[17:57:34 root@ob1-rocky-150 log]# grep 'YB42C0A8A795-00061DCE2B3D7A8D-0-0' observer.log.20240920175656086 [2024-09-20 17:54:31.013904] INFO [SHARE] add_event (ob_event_history_table_operator.h:289) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=49] event table add task(ret=0, event_table_name="__all_server_event_history", sql=INSERT INTO __all_server_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, svr_ip, svr_port) VALUES (usec_to_time(1726826071013840), 'storage_ha', 'schedule_ls_add start', 'tenant_id', 1008, 'ls_id', 1005, 'data_src', '"192.168.167.148:2882"', 'dest', '"192.168.167.150:2882"', '192.168.167.150', 2882)) [2024-09-20 17:54:31.013963] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:915) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=53][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1005}) [2024-09-20 17:54:31.014047] INFO [STORAGE.TRANS] alloc (ob_trans_factory.cpp:175) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=11] ObLSTxCtxMgr statistics(alloc_count=8682, release_count=8675, used=7) [2024-09-20 17:54:31.016000] INFO [STORAGE.TRANS] alloc (ob_trans_factory.cpp:183) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=38] alloc ls tx ctx mgr(partition_trans_ctx_mgr=0x14f604a04030) [2024-09-20 17:54:31.016059] INFO [STORAGE.TRANS] init (ob_trans_ctx_mgr_v4.cpp:198) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=18] ObLSTxCtxMgr inited success(this=0x14f604a04030, ls_id={id:1005}) [2024-09-20 17:54:31.016073] INFO [STORAGE.TRANS] create_ls (ob_trans_ctx_mgr_v4.cpp:2273) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=12] create ls success(tenant_id=1008, total_alloc=8594, total_release=8588, ls_id={id:1005}, ls_tx_ctx_mgr=0x14f604a04030) [2024-09-20 17:54:31.016085] INFO [STORAGE.TRANS] create_ls (ob_trans_service_v4.cpp:86) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=10] create ls success(tenant_id_=1008, ls_id={id:1005}) [2024-09-20 17:54:31.016148] INFO [STORAGE] set_allow_to_read_ (ob_ls_tablet_service.cpp:5885) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=16] set ls do not allow to read(ls={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:0, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:false, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:false}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}, migration_status=1, restore_status={status:0}) [2024-09-20 17:54:31.016269] INFO [LIB] ObSliceAlloc (ob_slice_alloc.h:318) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=93] ObSliceAlloc init finished(bsize_=7936, isize_=128, slice_limit_=7536, tmallocator_=NULL) [2024-09-20 17:54:31.016305] INFO [STORAGE] init (ob_tx_data_table.cpp:82) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=16] tx data table init success(sizeof(ObTxData)=104, sizeof(ObTxDataLinkNode)=8, this={this:0x14f5bc65d090, is_inited:true, is_started:false, ls_id:{id:1005}, tablet_id:{id:49402}, calc_upper_info:{min_start_scn_in_ctx_:{val:0, v:0}, keep_alive_scn_:{val:0, v:0}, update_ts_:0}, memtables_cache:{memtable_head_:-1, memtable_tail_:-1, memtable_handles_:[]}, ls:0x14f5bc656150, ls_tablet_svr:0x14f5bc656190, memtable_mgr:0x14f5bc6561b0, tx_ctx_table:0x14f5bc65c3e8}) [2024-09-20 17:54:31.016328] INFO [STORAGE.TRANS] acquire_ref_ (ob_tx_ctx_table.cpp:265) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=20] get ls tx ctx mgr successfully(txs=0x14f58d004030) [2024-09-20 17:54:31.016333] INFO [STORAGE] init (ob_tx_table.cpp:61) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=4] init tx table successfully(ret=0, ls->get_ls_id()={id:1005}) [2024-09-20 17:54:31.016359] INFO [CLOG] init (ob_garbage_collector.cpp:393) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=5] GC handler init success(ret=0, ls->get_ls_id()={id:1005}) [2024-09-20 17:54:31.016364] INFO [STORAGE] init (ob_ls_wrs_handler.cpp:39) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=5] ObLSWRSHandler init success(*this={is_inited:true, is_enabled:true, ls_id:{id:1005}, ls_weak_read_ts:{val:0, v:0}}) [2024-09-20 17:54:31.016377] INFO [STORAGE] init (ob_ls_reserved_snapshot_mgr.cpp:62) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] success to init snapshot mgr(ret=0, ls=0x14f5bc656150, ls_id={id:1005}, this=0x14f5bc664e90) [2024-09-20 17:54:31.016399] INFO [STORAGE] init (ob_ls_member_list_service.cpp:49) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=12] success to init member list service(ret=0, ls=0x14f5bc656150, ls_id={id:1005}, this=0x14f5bc665310) [2024-09-20 17:54:31.016414] INFO [STORAGE] init (ob_ls_block_tx_service.cpp:49) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=14] success to init block tx service(ret=0, ls=0x14f5bc656150, ls_id={id:1005}, this=0x14f5bc665450) [2024-09-20 17:54:31.016442] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] register_handler success(ret=0, type=1, role_change_handler=0x14f5bc65b258) [2024-09-20 17:54:31.016450] INFO [STORAGE] init (ob_ls.cpp:207) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=8] register to logservice success(logservice::TRANS_SERVICE_LOG_BASE_TYPE=1, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016475] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=12] register_handler success(ret=0, type=3, role_change_handler=0x14f5bc656198) [2024-09-20 17:54:31.016485] INFO [STORAGE] init (ob_ls.cpp:208) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=6] register to logservice success(logservice::STORAGE_SCHEMA_LOG_BASE_TYPE=3, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016506] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=15] register_handler success(ret=0, type=4, role_change_handler=0x14f5bc663350) [2024-09-20 17:54:31.016513] INFO [STORAGE] init (ob_ls.cpp:209) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=5] register to logservice success(logservice::TABLET_SEQ_SYNC_LOG_BASE_TYPE=4, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016563] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=24] register_handler success(ret=0, type=5, role_change_handler=0x14f5bc663398) [2024-09-20 17:54:31.016576] INFO [STORAGE] init (ob_ls.cpp:210) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=11] register to logservice success(logservice::DDL_LOG_BASE_TYPE=5, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016596] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] register_handler success(ret=0, type=6, role_change_handler=0x14f5bc663fe0) [2024-09-20 17:54:31.016604] INFO [STORAGE] init (ob_ls.cpp:211) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] register to logservice success(logservice::KEEP_ALIVE_LOG_BASE_TYPE=6, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016629] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=12] register_handler success(ret=0, type=9, role_change_handler=0x14f5bc65bdd8) [2024-09-20 17:54:31.016644] INFO [STORAGE] init (ob_ls.cpp:212) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=15] register to logservice success(logservice::GC_LS_LOG_BASE_TYPE=9, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016664] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=8] register_handler success(ret=0, type=27, role_change_handler=0x14f5bc663328) [2024-09-20 17:54:31.016684] INFO [STORAGE] init (ob_ls.cpp:213) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=19] register to logservice success(logservice::OBJ_LOCK_GARBAGE_COLLECT_SERVICE_LOG_BASE_TYPE=27, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016708] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=16] register_handler success(ret=0, type=17, role_change_handler=0x14f5bc665298) [2024-09-20 17:54:31.016719] INFO [STORAGE] init (ob_ls.cpp:214) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=9] register to logservice success(logservice::RESERVED_SNAPSHOT_LOG_BASE_TYPE=17, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016741] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=14] register_handler success(ret=0, type=18, role_change_handler=0x14f5bc6652c0) [2024-09-20 17:54:31.016750] INFO [STORAGE] init (ob_ls.cpp:215) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=9] register to logservice success(logservice::MEDIUM_COMPACTION_LOG_BASE_TYPE=18, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016765] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=8] register_handler success(ret=0, type=35, role_change_handler=0x14f5bc668c60) [2024-09-20 17:54:31.016772] INFO [STORAGE] init (ob_ls.cpp:216) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=6] register to logservice success(logservice::TRANSFER_HANDLER_LOG_BASE_TYPE=35, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016786] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] register_handler success(ret=0, type=37, role_change_handler=0x14f5bc665458) [2024-09-20 17:54:31.016794] INFO [STORAGE] init (ob_ls.cpp:217) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] register to logservice success(logservice::LS_BLOCK_TX_SERVICE_LOG_BASE_TYPE=37, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016820] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] register_handler success(ret=0, type=26, role_change_handler=0x14f5bc664118) [2024-09-20 17:54:31.016831] INFO [STORAGE] ls_init_for_dup_table_ (ob_ls.cpp:346) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] register to logservice success(logservice::DUP_TABLE_LOG_BASE_TYPE=26, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016853] INFO create_tg_tenant (thread_mgr.h:1043) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=9] create tg succeed(tg_id=595, tg=0x14f6371f1e60, thread_cnt=1, tg->attr_={name:TTLTabletMgr, type:3}, tg=0x14f6371f1e60) [2024-09-20 17:54:31.016871] INFO [SERVER] alloc_tenant_info (ob_tenant_tablet_ttl_mgr.cpp:625) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=13] finish create tenant ttl task(tenant_id=1008, ret=0, ls=NULL) [2024-09-20 17:54:31.016880] INFO [SERVER] init (ob_tenant_tablet_ttl_mgr.cpp:76) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=9] finish init tenant_tablet_ttl_mgr(tenant_id=1008, ret=0, ls=NULL) [2024-09-20 17:54:31.016887] INFO [SERVER] init (ob_tenant_tablet_ttl_mgr.cpp:51) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] success to init tenant ttl mgr(MTL_ID()=1008) [2024-09-20 17:54:31.016902] INFO [CLOG] register_handler (ob_role_change_handler.cpp:57) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] register_handler success(ret=0, type=39, role_change_handler=0x14f5bc6655a0) [2024-09-20 17:54:31.016910] INFO [STORAGE] init (ob_ls.cpp:322) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] register to logservice success(logservice::TTL_LOG_BASE_TYPE=39, ls_meta_.ls_id_={id:1005}) [2024-09-20 17:54:31.016917] INFO [STORAGE] init (ob_ls.cpp:323) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] register tenant tablet ttl mgr complete(ret=0, ret="OB_SUCCESS") [2024-09-20 17:54:31.016925] INFO [STORAGE] init (ob_ls.cpp:332) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] ls init success(ls_id={id:1005}) [2024-09-20 17:54:31.016967] INFO [STORAGE] add_ls (ob_ls_map.cpp:198) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=29] ls map add ls(ls_id={id:1005}, &ls=0x14f5bc656150, ref=0) [2024-09-20 17:54:31.016987] INFO [STORAGE] add_ls (ob_ls_map.cpp:231) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=16] ls map finish add ls(ls_id={id:1005}, &ls=0x14f5bc656150, ref=1, ret=0) [2024-09-20 17:54:31.028350] WDIAG [PALF] create_palf_handle_impl_ (palf_env_impl.cpp:407) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=10][errcode=-4264] PalfEnv can not hold more instance(ret=-4264, this={IPalfEnvImpl:{IPalfEnvImpl:"Dummy"}, self:"192.168.167.150:2882", log_dir:"/home/admin/oceanbase/store/ob_cluster1/clog/tenant_1008", disk_options_wrapper:{disk_opts_for_stopping_writing:{log_disk_size(MB):2560, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):2560, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):620, sequence:1}, log_alloc_mgr_:{flying_log_task:0, flying_meta_task:0}}, palf_id=1005) [2024-09-20 17:54:31.028486] INFO [PALF] create_palf_handle_impl_ (palf_env_impl.cpp:440) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=135] PalfEnvImpl create_palf_handle_impl_ finished(ret=-4264, palf_id=1005, access_mode=2, palf_base_info={prev_log_info:{log_id:0, lsn:{lsn:0}, scn:{val:0, v:0}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:0}}, replica_type=1, this={IPalfEnvImpl:{IPalfEnvImpl:"Dummy"}, self:"192.168.167.150:2882", log_dir:"/home/admin/oceanbase/store/ob_cluster1/clog/tenant_1008", disk_options_wrapper:{disk_opts_for_stopping_writing:{log_disk_size(MB):2560, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):2560, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):620, sequence:1}, log_alloc_mgr_:{flying_log_task:0, flying_meta_task:0}}) [2024-09-20 17:54:31.028524] WDIAG [PALF] create (palf_env.cpp:110) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=35][errcode=-4264] create_palf_handle_impl failed(ret=-4264, palf_id=1005) [2024-09-20 17:54:31.028629] WDIAG [CLOG] create_ls_ (ob_log_service.cpp:662) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=103][errcode=-4264] failed to get palf_handle(ret=-4264, id={id:1005}, replica_type=0) [2024-09-20 17:54:31.028642] WDIAG [CLOG] create_ls (ob_log_service.cpp:340) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=12][errcode=-4264] create ls failed(ret=-4264, id={id:1005}, replica_type=0, tenant_role={value:3}, palf_base_info={prev_log_info:{log_id:0, lsn:{lsn:0}, scn:{val:0, v:0}, log_proposal_id:9223372036854775807, accum_checksum:-1}, curr_lsn:{lsn:0}}) [2024-09-20 17:54:31.028661] WDIAG [STORAGE] create_ls (ob_ls.cpp:412) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=17][errcode=-4264] create palf failed(ret=-4264, ls_meta={tenant_id:1008, ls_id:{id:1005}, ls_create_status:0, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}) [2024-09-20 17:54:31.028700] WDIAG [STORAGE] create_ls_for_ha (ob_ls_service.cpp:1129) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=38][errcode=-4264] enable ls palf failed(ret=-4264, ls_meta={tenant_id:1008, ls_id:{id:1005}, ls_create_status:0, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}) [2024-09-20 17:54:31.039368] INFO [STORAGE] del_ls (ob_ls_map.cpp:265) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=28] ls service del ls(ls_id={id:1005}, ls=0x14f5bc656150, ref=1) [2024-09-20 17:54:31.039413] INFO [STORAGE] revert_ls (ob_ls_map.h:114) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=83] ObLSMap free ls(ls=0x14f5bc656150, mod=15, ls->get_ls_id()={id:1005}) [2024-09-20 17:54:31.039436] INFO [STORAGE] destroy (ob_ls.cpp:755) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=8] ObLS destroy(this=0x14f5bc656150, *this={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}, lbt()="0x113aea2c 0xd37ca3a 0xd35c3e4 0xd35b9d0 0xd489555 0xd493873 0xd49bd2e 0x964f6a1 0x4c450ea 0x488a217 0x4883944 0x92a13b4 0x116659bf 0x14f67f6fb1ca 0x14f67f367e73") [2024-09-20 17:54:31.039545] INFO [STORAGE] offline_advance_epoch_ (ob_ls.cpp:1076) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=107] offline not advance epoch(maybe repeat call)(ret=0, ls_meta_={tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch=0) [2024-09-20 17:54:31.039598] INFO [STORAGE] offline (ob_ls_restore_handler.cpp:113) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=48] ls restore handler offline finish [2024-09-20 17:54:31.039609] INFO [PALF] offline (ob_log_handler.cpp:1641) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=9] ObLogHandler has already been destroyed(ret=0, this={role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}) [2024-09-20 17:54:31.039733] INFO [STORAGE] offline (ob_tablet_gc_service.cpp:732) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=10] tablet gc handler offline(this={tablet_persist_trigger:3, is_inited:true}, ls_={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}, ls_->get_ls_meta()={tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}) [2024-09-20 17:54:31.039822] INFO [COMMON] check_ls_compaction_dag_exist_with_cancel (ob_dag_scheduler.cpp:2294) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=84] cancel dag when check ls compaction dag exist(ret=0, ret="OB_SUCCESS", cancel_dag_cnt=0, exist=false) [2024-09-20 17:54:31.039833] INFO [STORAGE] offline (ob_ls_wrs_handler.cpp:61) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=10] weak read handler disabled(*this={is_inited:true, is_enabled:false, ls_id:{id:1005}, ls_weak_read_ts:{val:1, v:0}}) [2024-09-20 17:54:31.039856] INFO [STORAGE] offline (ob_ls_ddl_log_handler.cpp:204) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=9] ddl log hanlder offline(ret=0, ls_meta={tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}) [2024-09-20 17:54:31.039878] INFO [STORAGE.TRANS] switch_state (ob_trans_ctx_mgr_v4.cpp:361) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=14] ObLSTxCtxMgr switch state success(ls_id=1005, F_WORKING ~> F_ALL_BLOCKED, op=BLOCK_ALL) [2024-09-20 17:54:31.039904] INFO [STORAGE.TRANS] block_all (ob_trans_ctx_mgr_v4.cpp:1025) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=26] block ls(ret=0, manager={this:0x14f604a04030, ls_id:{id:1005}, tenant_id:1008, state:"F_ALL_BLOCKED", total_tx_ctx_count:0, active_tx_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:18446744073709551615, v:3}, last_push_gc_task_ts_:1726826071014392, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:2}) [2024-09-20 17:54:31.039927] INFO [STORAGE.TRANS] prepare_offline (ob_ls_tx_service.cpp:683) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=22] prepare offline ls(ret=0, start_ts=1726826071039421, mgr_=0x14f604a04030, ls_id={id:1005}) [2024-09-20 17:54:31.039953] INFO [STORAGE.TRANS] prepare_offline (ob_tx_table.cpp:92) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=23] tx table prepare offline succeed(this={this:0x14f5bc65c350, is_inited:true, epoch:0, state:"PREPARE_OFFLINE", ls:0x14f5bc656150, tx_data_table:{this:0x14f5bc65d090, is_inited:true, is_started:false, ls_id:{id:1005}, tablet_id:{id:49402}, calc_upper_info:{min_start_scn_in_ctx_:{val:0, v:0}, keep_alive_scn_:{val:0, v:0}, update_ts_:0}, memtables_cache:{memtable_head_:-1, memtable_tail_:-1, memtable_handles_:[]}, ls:0x14f5bc656150, ls_tablet_svr:0x14f5bc656190, memtable_mgr:0x14f5bc6561b0, tx_ctx_table:0x14f5bc65c3e8}, mini_cache_hit_cnt:0, kv_cache_hit_cnt:0, read_tx_data_table_cnt:0}) [2024-09-20 17:54:31.040092] INFO [STORAGE.TRANS] switch_state (ob_trans_ctx_mgr_v4.cpp:361) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=131] ObLSTxCtxMgr switch state success(ls_id=1005, F_ALL_BLOCKED ~> F_ALL_BLOCKED, op=BLOCK_ALL) [2024-09-20 17:54:31.040100] INFO [STORAGE.TRANS] block_all (ob_trans_ctx_mgr_v4.cpp:1025) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=8] block ls(ret=0, manager={this:0x14f604a04030, ls_id:{id:1005}, tenant_id:1008, state:"F_ALL_BLOCKED", total_tx_ctx_count:0, active_tx_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:18446744073709551615, v:3}, last_push_gc_task_ts_:1726826071014392, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:2}) [2024-09-20 17:54:31.041967] INFO [STORAGE.TRANS] kill_all_tx (ob_trans_ctx_mgr_v4.cpp:991) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=19] [LsTxCtxMgr] kill_all_tx done(timeguard=time guard 'ctxmgr kill_all_tx' cost too much time, used=1860, manager={this:0x14f604a04030, ls_id:{id:1005}, tenant_id:1008, state:"F_ALL_BLOCKED", total_tx_ctx_count:0, active_tx_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:18446744073709551615, v:3}, last_push_gc_task_ts_:1726826071014392, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:2}) [2024-09-20 17:54:31.042023] WDIAG [STORAGE] offline_tx_ctx_table_ (ob_tx_table.cpp:107) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=47][errcode=-4725] get tablet failed(ret=-4725) [2024-09-20 17:54:31.042036] WDIAG [STORAGE] release_memtables (ob_i_memtable_mgr.cpp:183) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=11][errcode=-4006] not inited(ret=-4006) [2024-09-20 17:54:31.042045] WDIAG [STORAGE] offline (ob_tx_data_memtable_mgr.cpp:96) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7][errcode=-4006] release tx data memtables failed(ret=-4006, ret="OB_NOT_INIT") [2024-09-20 17:54:31.042054] WDIAG [STORAGE] offline (ob_tx_data_table.cpp:221) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=8][errcode=0] release memtables failed(ret=0, ret="OB_SUCCESS") [2024-09-20 17:54:31.042061] INFO [STORAGE] offline (ob_tx_table.cpp:153) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] tx table offline succeed(ls_id_={id:1005}, this={this:0x14f5bc65c350, is_inited:true, epoch:0, state:"OFFLINE", ls:0x14f5bc656150, tx_data_table:{this:0x14f5bc65d090, is_inited:true, is_started:false, ls_id:{id:1005}, tablet_id:{id:49402}, calc_upper_info:{min_start_scn_in_ctx_:{val:0, v:0}, keep_alive_scn_:{val:0, v:0}, update_ts_:0}, memtables_cache:{memtable_head_:-1, memtable_tail_:-1, memtable_handles_:[]}, ls:0x14f5bc656150, ls_tablet_svr:0x14f5bc656190, memtable_mgr:0x14f5bc6561b0, tx_ctx_table:0x14f5bc65c3e8}, mini_cache_hit_cnt:0, kv_cache_hit_cnt:0, read_tx_data_table_cnt:0}) [2024-09-20 17:54:31.042105] INFO [STORAGE.TABLELOCK] offline (ob_lock_table.cpp:255) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=39] lock table offline(parent_->get_ls_id()={id:1005}) [2024-09-20 17:54:31.042118] WDIAG [STORAGE.TABLELOCK] offline (ob_lock_table.cpp:262) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=10][errcode=-4725] get tablet failed(ret=-4725) [2024-09-20 17:54:31.042220] INFO [STORAGE] offline (ob_empty_shell_task.cpp:369) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=6] tablet empty shell handler offline(this={is_inited:true, is_trigger:true, stopped:true}, ls_={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}, ls_->get_ls_meta()={tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}) [2024-09-20 17:54:31.042318] INFO [STORAGE] stop (ob_tx_table.cpp:85) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=96] tx table stop finish(this={this:0x14f5bc65c350, is_inited:true, epoch:0, state:"OFFLINE", ls:0x14f5bc656150, tx_data_table:{this:0x14f5bc65d090, is_inited:true, is_started:false, ls_id:{id:1005}, tablet_id:{id:49402}, calc_upper_info:{min_start_scn_in_ctx_:{val:0, v:0}, keep_alive_scn_:{val:0, v:0}, update_ts_:0}, memtables_cache:{memtable_head_:-1, memtable_tail_:-1, memtable_handles_:[]}, ls:0x14f5bc656150, ls_tablet_svr:0x14f5bc656190, memtable_mgr:0x14f5bc6561b0, tx_ctx_table:0x14f5bc65c3e8}, mini_cache_hit_cnt:0, kv_cache_hit_cnt:0, read_tx_data_table_cnt:0}) [2024-09-20 17:54:31.042340] INFO [STORAGE.DUP_TABLE] prepare_state_change (ob_dup_table_base.cpp:153) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=21] the cur state has already been same as target_state(ret=-4517, target_state=30, this={module_name:"DupTableLSHandler", cur_state_:{role_state_:10, offline_state_:20, stop_state_:30}}) [2024-09-20 17:54:31.042356] WDIAG [STORAGE.DUP_TABLE] stop (ob_dup_table_util.cpp:464) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=14][errcode=-4517] prepare state change failed(ret=-4517, this={ls_id_:{id:1005}, ls_state_helper_:{module_name:"DupTableLSHandler", cur_state_:{role_state_:10, offline_state_:20, stop_state_:30}}, dup_ls_ckpt_:{dup_ls_meta_:{ls_id_:{id:1005}, lease_log_applied_scn_:{val:18446744073709551615, v:3}, readable_tablets_base_scn_:{val:18446744073709551615, v:3}, readable_tablets_min_base_applied_scn_:{val:18446744073709551615, v:3}, lease_item_array_.count():0}, lease_log_rec_scn_:{val:18446744073709551615, v:3}, start_replay_scn_:{val:18446744073709551615, v:3}, readable_ckpt_base_scn_is_accurate_:true}, lease_mgr_ptr_:null, tablets_mgr_ptr_:null, ts_sync_mgr_ptr_:null, log_operator_:null, interface_stat_:{dup_table_follower_read_succ_cnt_:0, dup_table_follower_read_tablet_not_exist_cnt_:0, dup_table_follower_read_lease_expired_cnt_:0, dup_table_follower_read_lease_expired_cnt_:0, dup_table_redo_sync_succ_cnt_:0, dup_table_redo_sync_fail_cnt_:0}}) [2024-09-20 17:54:31.042432] INFO [CLOG] stop (ob_log_restore_handler.cpp:138) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=58] ObLogRestoreHandler stop(ret=0, this={is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}) [2024-09-20 17:54:31.042469] INFO [SERVER] stop (ob_tenant_tablet_ttl_mgr.cpp:164) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=26] tenant_tablet_ttl_mgr: begin to stop(tenant_id=1008, ls={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-09-20 17:54:31.042530] INFO [SERVER] stop (ob_tenant_tablet_ttl_mgr.cpp:178) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=59] tenant_tablet_ttl_mgr: finish to stop(ret=0, is_timer_start=false, tenant_id=1008, ls={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-09-20 17:54:31.042580] INFO [STORAGE] stop_ (ob_ls.cpp:611) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=46] stop_ls finish(ret=0, ret="OB_SUCCESS", this={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-09-20 17:54:31.042627] WDIAG [STORAGE] get_ls_migration_task_ (ob_ls_migration_handler.cpp:325) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=42][errcode=-4018] migration task is empty(ret=-4018, ls_={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-09-20 17:54:31.042964] INFO [STORAGE] prepare_for_safe_destroy (ob_tx_data_table.cpp:210) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=73] tx data table prepare for safe destroy(ret=0, ret="OB_SUCCESS", ls_id_={id:1005}) [2024-09-20 17:54:31.042995] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=19] unregister_handler success(type=1) [2024-09-20 17:54:31.043017] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=13] unregister_handler success(type=3) [2024-09-20 17:54:31.043036] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=6] unregister_handler success(type=4) [2024-09-20 17:54:31.043061] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=9] unregister_handler success(type=5) [2024-09-20 17:54:31.043089] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=12] unregister_handler success(type=6) [2024-09-20 17:54:31.043106] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=9] unregister_handler success(type=9) [2024-09-20 17:54:31.043136] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=8] unregister_handler success(type=27) [2024-09-20 17:54:31.043158] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=6] unregister_handler success(type=17) [2024-09-20 17:54:31.043180] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=6] unregister_handler success(type=18) [2024-09-20 17:54:31.043194] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] unregister_handler success(type=35) [2024-09-20 17:54:31.043221] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=16] unregister_handler success(type=37) [2024-09-20 17:54:31.043265] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=14] unregister_handler success(type=26) [2024-09-20 17:54:31.043305] INFO [CLOG] unregister_handler (ob_role_change_handler.cpp:67) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=16] unregister_handler success(type=39) [2024-09-20 17:54:31.043313] INFO [SERVER] destroy (ob_tenant_tablet_ttl_mgr.cpp:190) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] tenant_tablet_ttl_mgr: begin to destroy(ls={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}, tenant_id=1008) [2024-09-20 17:54:31.043403] INFO [SERVER] wait (ob_tenant_tablet_ttl_mgr.cpp:183) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=89] tenant_tablet_ttl_mgr: begin to wait(ls={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}, tenant_id=1008) [2024-09-20 17:54:31.043451] INFO [SERVER] wait (ob_tenant_tablet_ttl_mgr.cpp:185) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=46] tenant_tablet_ttl_mgr: finish to wait(ls={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}, tenant_id=1008) [2024-09-20 17:54:31.043500] INFO destroy_tg (thread_mgr.cpp:89) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=44] destroy tg(tg_id=595, tg=0x14f6371f1e60, tg->attr_={name:TTLTabletMgr, type:3}) [2024-09-20 17:54:31.043515] INFO [SERVER] destroy (ob_tenant_tablet_ttl_mgr.cpp:194) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=12] tenant_tablet_ttl_mgr: finish to destroy(ls={ls_meta:{tenant_id:1008, ls_id:{id:1005}, ls_create_status:2, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:9223372036854775807, palf_env_:null, is_in_stop_state_:true, is_inited_:false, id_:-1}, restore_handler:{is_inited:false, is_in_stop_state:true, id:-1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1310720, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:3, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}, tenant_id=1008) [2024-09-20 17:54:31.043562] INFO [STORAGE.TRANS] release_ref_ (ob_tx_ctx_table.cpp:283) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=42] revert ls tx ctx mgr successfully(ls_id_={id:1005}, this=0x14f5bc65c3e8) [2024-09-20 17:54:31.043776] INFO [STORAGE.TRANS] switch_state (ob_trans_ctx_mgr_v4.cpp:361) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=9] ObLSTxCtxMgr switch state success(ls_id=1005, F_ALL_BLOCKED ~> STOPPED, op=STOP) [2024-09-20 17:54:31.045427] INFO [STORAGE.TRANS] stop (ob_trans_ctx_mgr_v4.cpp:966) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=11] [LsTxCtxMgr] stop done(timeguard=time guard 'ctxmgr stop' cost too much time, used=1682, manager={this:0x14f604a04030, ls_id:{id:1005}, tenant_id:1008, state:"STOPPED", total_tx_ctx_count:0, active_tx_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:18446744073709551615, v:3}, last_push_gc_task_ts_:1726826071014392, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:2}) [2024-09-20 17:54:31.045473] INFO [STORAGE.TRANS] stop_ls_ (ob_trans_ctx_mgr_v4.cpp:1619) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=45] stop ls success(ls_id={id:1005}, ctx_count=0) [2024-09-20 17:54:31.045492] INFO [STORAGE.TRANS] wait_ls_ (ob_trans_ctx_mgr_v4.cpp:1652) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=14] wait ls success(ls_id={id:1005}) [2024-09-20 17:54:31.045503] INFO [STORAGE.TRANS] remove_ls (ob_trans_ctx_mgr_v4.cpp:2350) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=8] remove ls success(ls_tx_ctx_mgr=0x14f604a04030, ls_id={id:1005}, total_alloc=8594, total_release=8589) [2024-09-20 17:54:31.045517] INFO [STORAGE.TRANS] free_value (ob_trans_ctx_mgr_v4.h:891) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=13] ObLSTxCtxMgr release(*p={this:0x14f604a04030, ls_id:{id:1005}, tenant_id:1008, state:"STOPPED", total_tx_ctx_count:0, active_tx_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:18446744073709551615, v:3}, last_push_gc_task_ts_:1726826071014392, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:0}) [2024-09-20 17:54:31.045570] INFO [STORAGE.TRANS] destroy (ob_trans_ctx_mgr_v4.cpp:209) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=25] ObLSTxCtxMgr destroyed(this=0x14f604a04030, ls_id={id:1005}) [2024-09-20 17:54:31.047412] INFO [STORAGE.TRANS] release (ob_trans_factory.cpp:195) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=21] release ls tx ctx mgr(partition_trans_ctx_mgr=0x14f604a04030) [2024-09-20 17:54:31.047449] INFO [STORAGE.TRANS] remove_ls (ob_trans_ctx_mgr_v4.cpp:2364) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=36] remove ls(ret=0, ret="OB_SUCCESS", ls_id={id:1005}, graceful=false) [2024-09-20 17:54:31.047464] INFO [STORAGE.TRANS] remove_ls (ob_trans_service_v4.cpp:116) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=12] remove ls success(ls_id={id:1005}, graceful=false) [2024-09-20 17:54:31.047473] INFO [STORAGE.TRANS] remove_ls (ob_trans_service_v4.cpp:121) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=8] remove ls success(ls_id={id:1005}, graceful=false) [2024-09-20 17:54:31.047674] INFO [STORAGE] create_ls_for_ha (ob_ls_service.cpp:1172) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=7] create_ls for ha finish(ret=-4264, ls_meta={tenant_id:1008, ls_id:{id:1005}, ls_create_status:0, clog_checkpoint_scn:{val:0, v:0}, clog_base_lsn:{lsn:0}, rebuild_seq:0, migration_status:1, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:0, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, abs_timeout_ts=1726826080013468) [2024-09-20 17:54:31.047740] WDIAG [COMMON] process (ob_rpc_processor_simple.cpp:217) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=63][errcode=-4264] failed to create ls for ha(ret=-4264, arg_={task_id:YB42C0A8A795-00061DCE2B3D7A8D-0-0, tenant_id:1008, ls_id:{id:1005}, dst:{server:"192.168.167.150:2882", timestamp:1726826070954666, flag:0, replica_type:0, region:"default_region", memstore_percent:100}, data_source:{server:"192.168.167.148:2882", timestamp:1, flag:0, replica_type:0, region:"default_region", memstore_percent:100}, orig_paxos_replica_number:3, new_paxos_replica_number:3, skip_change_member_list:false}, migration_op_arg={ls_id:{id:1005}, type:"ADD_LS_OP", cluster_id:10002, priority:0, src:{server:"192.168.167.148:2882", timestamp:1, flag:0, replica_type:0, region:"default_region", memstore_percent:100}, dst:{server:"192.168.167.150:2882", timestamp:1726826070954666, flag:0, replica_type:0, region:"default_region", memstore_percent:100}, data_src:{server:"192.168.167.148:2882", timestamp:1, flag:0, replica_type:0, region:"default_region", memstore_percent:100}, paxos_replica_number:3}) [2024-09-20 17:54:31.047890] INFO [SHARE] add_event (ob_event_history_table_operator.h:289) [3301570][T1008_L0_G0][T1008][YB42C0A8A795-00061DCE2B3D7A8D-0-0] [lt=82] event table add task(ret=0, event_table_name="__all_server_event_history", sql=INSERT INTO __all_server_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, svr_ip, svr_port) VALUES (usec_to_time(1726826071047822), 'storage_ha', 'schedule_ls_add failed', 'tenant_id', 1008, 'ls_id', '{id:1005}', 'result', -4264, '192.168.167.150', 2882))