[2024-08-16 10:19:54.439091] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] tx gc loop thread is running(MTL_ID()=1002) [2024-08-16 10:19:54.440008] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774792469884003, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:19:54.440098] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=86] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:54.440645] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=27][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:54.441101] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1260}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202608224000, v:0}, clog_base_lsn:{lsn:8723619840}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774792469884003, v:0}, tablet_change_checkpoint_scn:{val:1723719123293042000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723639313794159001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1260, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:19:54.441247] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=60] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:54.441959] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=37410819, this={dio_aligned_buf:{buf_write_offset:2164, buf_padding_size:1932, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:45196054, truncate_used_ts:11123492}, log_block_size:67108864, dir_fd:196, io_fd:111}, aligned_buf_len=4096, aligned_buf="éì njŽû", aligned_block_offset=37408768, buf_len=113, total_write_size_=14678, total_write_size_after_dio_=438272, ob_pwrite_used_ts=62886, count=103, each_pwrite_cost=610) [2024-08-16 10:19:54.442378] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1261}, ls_create_status:1, clog_checkpoint_scn:{val:1723752188579788000, v:0}, clog_base_lsn:{lsn:32545812480}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774792469884003, v:0}, tablet_change_checkpoint_scn:{val:1723719128263216001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723617733349446001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:4, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1261}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1261, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:19:54.442444] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=63] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:54.443476] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774792469884003, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:19:54.443528] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=50] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:54.446956] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.446976] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:54.446986] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.446995] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.447009] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.451106] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=38] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=787, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:19:54.457399] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=43] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=43, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:19:54.458247] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF37-0-0] [lt=34][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509793, log_task={header:{begin_lsn:{lsn:18544212514}, end_lsn:{lsn:18544212636}, log_id:53509793, min_scn:{val:1723774793871611004, v:0}, max_scn:{val:1723774793871611004, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544212392}, prev_proposal_id:7, committed_end_lsn:{lsn:18544212514}, data_checksum:1913811101, accum_checksum:3601298832, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774794357928, freeze_ts:1723774794357928, submit_ts:1723774794357932, flushed_ts:1723774794358603, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:671}, fs_cb_begin_ts=1723774794458237, log_life_time=100309) [2024-08-16 10:19:54.458910] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B5-0-0] [lt=22] succeed to calculate base table sharding info(table_id=18446744073709551614, ref_table_id=12008, is_global_index=false) [2024-08-16 10:19:54.458966] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B5-0-0] [lt=26] total rowcount, use statistics(table_meta_info_.table_row_count_=2, table_meta_info_.average_row_size_=4.000000000000000000e+01, table_meta_info_.micro_block_count_=0, table_meta_info_.part_size_=8.000000000000000000e+01) [2024-08-16 10:19:54.460650] WDIAG [SQL.PC] common_free (ob_lib_cache_object_manager.cpp:141) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B5-0-0] [lt=24][errcode=0] set logical del time(cache_obj->get_logical_del_time()=10451106719571, cache_obj->added_lc()=false, cache_obj->get_object_id()=254217, cache_obj->get_tenant_id()=1002, lbt()="0x12a0688c 0xaa7c9e4 0x4e44d8a 0x4e36df8 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:54.466472] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=797458432, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:54.466722] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash time detail, (compute_wash_size_time=112, refresh_score_time=64, wash_time=16) [2024-08-16 10:19:54.470197] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=3019872, clean_num=62914) [2024-08-16 10:19:54.475638] WDIAG [SQL.RESV] find_column_in_joined_table (ob_column_namespace_checker.cpp:482) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=22][errcode=-5207] column in joined table is ambiguous(q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:54.475704] WDIAG [SQL.RESV] find_column_in_joined_table (ob_column_namespace_checker.cpp:468) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=64][errcode=-5207] find column in left table failed(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:54.475731] WDIAG [SQL.RESV] find_column_in_joined_table (ob_column_namespace_checker.cpp:468) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=26][errcode=-5207] find column in left table failed(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:54.475751] WDIAG [SQL.RESV] find_column_in_joined_table (ob_column_namespace_checker.cpp:468) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=19][errcode=-5207] find column in left table failed(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:54.475771] WDIAG [SQL.RESV] find_column_in_joined_table (ob_column_namespace_checker.cpp:468) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=19][errcode=-5207] find column in left table failed(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:54.475793] WDIAG [SQL.RESV] check_table_column_namespace (ob_column_namespace_checker.cpp:103) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=21][errcode=-5207] find column in table failed(ret=-5207) [2024-08-16 10:19:54.475803] WDIAG [SQL.RESV] resolve_table_column_expr (ob_dml_resolver.cpp:3115) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=6][errcode=-5207] column not found in table(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:54.475840] WDIAG [SQL.RESV] resolve_table_column_ref (ob_select_resolver.cpp:5023) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=23][errcode=-5207] resolve table column expr failed(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}, lbt()="0x12a0688c 0xd8c56b9 0xd8c5596 0xd8c3ac9 0xd8c48ec 0xd6fefc2 0xd6f888d 0xd683d91 0xd8b7a20 0xd889202 0xd898997 0xd7c0576 0xd73a576 0xd739d50 0xd6e5e94 0xd884fec 0xd898997 0xcc3a2d5 0x51eb425 0x504b6d2 0x5044bba 0x5042480 0x4e55d67 0x4e3568a 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:54.481585] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=25] succeed to calculate base table sharding info(table_id=18446744073709551609, ref_table_id=516604, is_global_index=false) [2024-08-16 10:19:54.481620] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=21] total rowcount, use statistics(table_meta_info_.table_row_count_=806, table_meta_info_.average_row_size_=9.139000000000000000e+03, table_meta_info_.micro_block_count_=40, table_meta_info_.part_size_=7.366034000000000000e+06) [2024-08-16 10:19:54.486677] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=15] succeed to calculate base table sharding info(table_id=18446744073709551608, ref_table_id=525321, is_global_index=false) [2024-08-16 10:19:54.486706] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=19] total rowcount, use statistics(table_meta_info_.table_row_count_=12926, table_meta_info_.average_row_size_=7.960000000000000000e+02, table_meta_info_.micro_block_count_=55, table_meta_info_.part_size_=1.028909600000000000e+07) [2024-08-16 10:19:54.488810] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=13] succeed to calculate base table sharding info(table_id=18446744073709551607, ref_table_id=559954, is_global_index=false) [2024-08-16 10:19:54.488836] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=18] total rowcount, use statistics(table_meta_info_.table_row_count_=768506, table_meta_info_.average_row_size_=1.628000000000000000e+03, table_meta_info_.micro_block_count_=3774, table_meta_info_.part_size_=1.251127768000000000e+09) [2024-08-16 10:19:54.489272] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=13] succeed to calculate base table sharding info(table_id=18446744073709551606, ref_table_id=545627, is_global_index=false) [2024-08-16 10:19:54.489300] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=19] total rowcount, use statistics(table_meta_info_.table_row_count_=140460, table_meta_info_.average_row_size_=2.716000000000000000e+03, table_meta_info_.micro_block_count_=6132, table_meta_info_.part_size_=3.814893600000000000e+08) [2024-08-16 10:19:54.489804] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=10] succeed to calculate base table sharding info(table_id=18446744073709551605, ref_table_id=583346, is_global_index=false) [2024-08-16 10:19:54.489833] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=19] total rowcount, use statistics(table_meta_info_.table_row_count_=144, table_meta_info_.average_row_size_=1.800000000000000000e+02, table_meta_info_.micro_block_count_=1, table_meta_info_.part_size_=2.592000000000000000e+04) [2024-08-16 10:19:54.490066] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=10] succeed to calculate base table sharding info(table_id=18446744073709551604, ref_table_id=500059, is_global_index=false) [2024-08-16 10:19:54.490109] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=35] total rowcount, use statistics(table_meta_info_.table_row_count_=3519, table_meta_info_.average_row_size_=1.380000000000000000e+02, table_meta_info_.micro_block_count_=3, table_meta_info_.part_size_=4.856220000000000000e+05) [2024-08-16 10:19:54.495320] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.495340] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:54.495350] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.495359] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.495373] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.507266] INFO [SQL.PC] inner_add_cache_obj (ob_pcv_set.cpp:266) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=12] has identical pcv(is_same=true, pcv=0x7fa704562ae0) [2024-08-16 10:19:54.508863] WDIAG [SQL.ENG] set_sqcs_accessed_location (ob_px_util.cpp:1145) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=21][errcode=-4016] we do not find this addr's execution sqc(ret=-4016, n_locations=0, locations.size()=1, sqcs=[{need_report:false, execution_id:338495197, qc_id:1, sqc_id:0, dfo_id:1, exec_addr:"192.168.0.124:2882", qc_addr:"192.168.0.128:2882", qc_ch_info:{chid:0, type:0, peer:"0.0.0.0:0", role:0, tenant_id:0, state_:0}, sqc_ch_info:{chid:0, type:0, peer:"0.0.0.0:0", role:0, tenant_id:0, state_:0}, task_count:0, max_task_count:1, min_task_count:1, thread_inited:false, thread_finish:false, px_int_id:{query_interrupt_id:task(1720514724410945,68719479205), px_interrupt_id:task(1720514724410945,68723677645)}, transmit_use_interm_result:false, recieve_use_interm_result:false, serial_receive_channels:[], sqc_order_gi_tasks:false}], locations=[{tablet_id:{id:1152921504606866871}, ls_id:{id:1260}, server:"192.168.0.128:2882", in_retry:0, partition_id:524323, first_level_part_id:18446744073709551615}]) [2024-08-16 10:19:54.508919] WDIAG [SQL.ENG] set_dfo_accessed_location (ob_px_util.cpp:1050) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=53][errcode=-4016] failed to set sqc accessed location(ret=-4016, table_location_key=18446744073709551609, ref_table_id=524299, table_loc={loc_meta:{table_loc_id:18446744073709551609, ref_table_id:524299, related_table_ids:[516604], use_dist_das:0, select_leader:1, is_dup_table:0, is_weak_read:0, unuse_related_pruning:0, is_external_table:0}, tablet_locs:[{tablet_id:{id:1152921504606866871}, ls_id:{id:1260}, server:"192.168.0.128:2882", in_retry:0, partition_id:524323, first_level_part_id:18446744073709551615}], is_writing:0, is_reading:0, need_refresh:0, is_fk_check:0}) [2024-08-16 10:19:54.508942] WDIAG [SQL.ENG] alloc_by_data_distribution_inner (ob_px_util.cpp:382) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=21][errcode=-4016] fail to set all table partition for tsc(ret=-4016, scan_ops.count()=3, dml_op=NULL, table_location_key=18446744073709551604, ref_table_id=500061, locations=[{tablet_id:{id:1152921504606847007}, ls_id:{id:1024}, server:"192.168.0.124:2882", in_retry:0, partition_id:500061, first_level_part_id:18446744073709551615}]) [2024-08-16 10:19:54.508952] WDIAG [SQL.ENG] alloc_by_data_distribution (ob_px_util.cpp:63) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=8][errcode=-4016] failed to alloc data distribution(ret=-4016) [2024-08-16 10:19:54.508957] WDIAG [SQL.ENG] init_all_dfo_channel (ob_dfo_scheduler.cpp:321) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=5][errcode=-4016] fail to alloc data distribution(ret=-4016) [2024-08-16 10:19:54.508961] WDIAG [SQL.ENG] startup_msg_loop (ob_px_scheduler.cpp:128) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=4][errcode=-4016] fail to init all dfo channel(ret=-4016) [2024-08-16 10:19:54.508966] WDIAG [SQL.ENG] fetch_rows (ob_px_fifo_coord_op.cpp:135) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=4][errcode=-4016] initial dfos NOT dispatched successfully(ret=-4016) [2024-08-16 10:19:54.508972] WDIAG [SQL.ENG] fetch_rows (ob_px_fifo_coord_op.cpp:244) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=3][errcode=-4016] QC get error code(ret=-4016, ret_terminate=0) [2024-08-16 10:19:54.508977] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=4][errcode=-4016] get next batch failed(ret=-4016, eval_ctx={batch_idx:0, batch_size:256, max_batch_size:256, frames_:0x7fa64260ed40}, id=0, op_name="PHY_PX_FIFO_COORD") [2024-08-16 10:19:54.508993] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1499) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=14][errcode=-4016] get next batch failed(ret=-4016) [2024-08-16 10:19:54.508999] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=5][errcode=-4016] get next row from exec result failed(ret=-4016) [2024-08-16 10:19:54.509005] WDIAG [SERVER] response_query_result (ob_query_driver.cpp:299) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=4][errcode=-4016] fail to iterate and response(ret=-4016, row_num=0, can_retry=true) [2024-08-16 10:19:54.509011] WDIAG [SERVER] response_result (ob_sync_plan_driver.cpp:101) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=5][errcode=-4016] response query result fail(ret=-4016) [2024-08-16 10:19:54.509017] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=4][errcode=-4016] [RETRY] check if need retry(v={force_local_retry:false, stmt_retry_times:0, local_retry_times:0, err_:-4016, err_:"OB_ERR_UNEXPECTED", retry_type:0, client_ret:-4016}, need_retry=false) [2024-08-16 10:19:54.509037] WDIAG [SERVER] response_result (ob_sync_plan_driver.cpp:113) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=19][errcode=-4016] result response failed, check if need retry(ret=-4016, cli_ret=-4016, retry_ctrl_.need_retry()=0) [2024-08-16 10:19:54.509066] WDIAG [SQL.ENG] destroy_all_channel (ob_px_coord_op.cpp:672) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=7][errcode=-4201] fail unlink channel(qc_ci={chid:0, type:0, peer:"0.0.0.0:0", role:0, tenant_id:0, state_:0}, ret=-4201) [2024-08-16 10:19:54.509110] WDIAG [SQL.ENG] destroy_all_channel (ob_px_coord_op.cpp:672) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=42][errcode=-4201] fail unlink channel(qc_ci={chid:0, type:0, peer:"0.0.0.0:0", role:0, tenant_id:0, state_:0}, ret=-4201) [2024-08-16 10:19:54.509127] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=8] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x7fa7311c74a0, tx_id:{txid:0}, state:1, addr:"192.168.0.128:2882", tenant_id:1002, session_id:3225589831, assoc_session_id:3225589831, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:-1, tx_consistency_type:0, isolation:-1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1723774794460533, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:true, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:-1, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x7fa66c6a21f8, id:3225589831, deser:false, tenant:"epean", tenant_id:1002, effective_tenant:"epean", effective_tenant_id:1002, database:"ep_trade_test", user:"epean_trade_test@%", consistency_level:3, session_state:2, autocommit:true, tx:0x7fa7311c74a0}, exec_ctx.get_errcode()=-4016) [2024-08-16 10:19:54.509298] WDIAG [SERVER] response_result (ob_sync_plan_driver.cpp:120) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=76][errcode=-4016] close result set fail(cret=-4016) [2024-08-16 10:19:54.509320] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:366) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=13] sending error packet(ob_error=-4016, client error=-4016, extra_err_info=NULL, lbt()="0x12a0688c 0xa2161e0 0xa1c51c6 0x4e7e290 0x4e35890 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:54.509363] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:549) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=8] dump txn free route audit_record(value=1048581, session->get_sessid()=3225589831, session->get_proxy_sessid()=13882346384134438935) [2024-08-16 10:19:54.509380] WDIAG [SERVER] do_process (obmp_query.cpp:823) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B6-0-0] [lt=14][errcode=-4016] execute query fail(ret=-4016, timeout_timestamp=1723784794472889) [2024-08-16 10:19:54.524566] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=23] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=6818, mem_used=12478464) [2024-08-16 10:19:54.527889] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [31738][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=54] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:940, allocator used:3737440, allocator total:3859072}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:140, allocator used:9182740, allocator total:9182740}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=13041812, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:63, allocator used:124992, allocator total:130816}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:19:54.532050] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=32] start do ls ha handler(ls_id_array_=[{id:1262}, {id:1079}, {id:1261}, {id:1260}]) [2024-08-16 10:19:54.533424] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 251.00) [2024-08-16 10:19:54.533455] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=25] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:54.533535] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=37] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.99, rt: 447.25) [2024-08-16 10:19:54.533555] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: MICRO_BLOCK_CACHE_IO, mode: read, size: 6436.57, iops: 27.90, rt: 240.18) [2024-08-16 10:19:54.533562] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:54.533588] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=20] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533600] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533607] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533612] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533617] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533622] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533626] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533629] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533634] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533639] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533644] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533648] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533654] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533661] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533677] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.533683] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:54.534803] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [31638][T1_Occam][T1][YB42C0A80080-00061CCCBF6D3E06-0-0] [lt=7] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:19:54.539946] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=9] trace max work area(auto_calc=true, tenant_max_memory_limit=1073741824, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=53687091, max_wa_memory_size=53687091, tmp_max_wa_memory_size=53687091, pre_mem_target=53687091, remain_memory_size=53687091, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=332681216, washable_size=-2, max_workarea_memory_size=53687091, max_tenant_memory_size=741060608, wash_ratio=6, tenant_id=1001) [2024-08-16 10:19:54.540026] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=38] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=6710886, calc_global_bound_size=1048576000000, mem_target_=53687091, wa_max_memory_size=53687091, profile_cnt_=0, total_memory_size=0, max_wa_size=53687091, avg_bound_size=6710886, max_bound_size=6710886, min_bound_size_=1048576) [2024-08-16 10:19:54.540089] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=39] timer to calc global bound size(ret=0, global_bound_size_=6710886, manual_calc_cnt_=0, drift_size_=0, pre_drift_size=0, wa_max_memory_size=53687091, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1001, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=6710886, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=53687091, auto_calc=true, sql_mem_callback_.get_total_dump_size()=0) [2024-08-16 10:19:54.540130] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:54.540162] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:54.540176] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:54.540184] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:54.540168] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=25] trace max work area(auto_calc=true, tenant_max_memory_limit=6442450944, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=3865470566, max_wa_memory_size=3865470566, tmp_max_wa_memory_size=3865470566, pre_mem_target=3865470566, remain_memory_size=3865470566, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=2457407488, washable_size=-2, max_workarea_memory_size=3865470566, max_tenant_memory_size=3985043456, wash_ratio=6, tenant_id=1002) [2024-08-16 10:19:54.540236] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=24] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=483183820, calc_global_bound_size=1048576000000, mem_target_=3865470566, wa_max_memory_size=3865470566, profile_cnt_=0, total_memory_size=0, max_wa_size=3865470566, avg_bound_size=483183820, max_bound_size=483183820, min_bound_size_=1048576) [2024-08-16 10:19:54.540255] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=17] timer to calc global bound size(ret=0, global_bound_size_=483183820, manual_calc_cnt_=19637, drift_size_=0, pre_drift_size=0, wa_max_memory_size=3865470566, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1002, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=483183820, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=3865470566, auto_calc=true, sql_mem_callback_.get_total_dump_size()=1112786688) [2024-08-16 10:19:54.540730] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=24][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:54.547040] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.547060] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:54.547089] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=27] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.547106] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.547123] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.548300] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A41-0-0] [lt=58] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:19:54.548986] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6201] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=21, stat_interval=1000000, avg (count/cost)=31, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:19:54.558668] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1623) [2024-08-16 10:19:54.590620] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=10][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:19:54.593040] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=22] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:19:54.593156] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=21][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:19:54.593176] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=19][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:19:54.593184] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=7] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:19:54.595401] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.595422] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:54.595436] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.595457] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.595488] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.604344] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=23] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:19:54.606412] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=15] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:19:54.610137] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [31774][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=10] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:19:54.610161] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:230) [31774][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=18] [tabletchange] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12636) [2024-08-16 10:19:54.616274] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=25] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60635, mem_used=135183360) [2024-08-16 10:19:54.623580] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=16] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=9) [2024-08-16 10:19:54.639251] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:54.639376] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:54.639395] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:54.639404] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:54.640505] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F734F9-0-0] [lt=24] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=8, avg_log_gen_to_submit_time=73, avg_log_submit_to_flush_time=708, avg_log_submit_to_slide_time=1198) [2024-08-16 10:19:54.640804] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=18][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:54.647088] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.647112] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:19:54.647123] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.647132] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.647148] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.649416] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F734FA-0-0] [lt=21] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=14, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:19:54.649437] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F734FA-0-0] [lt=20] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1977, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:19:54.658143] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF43-0-0] [lt=7][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509796, log_task={header:{begin_lsn:{lsn:18544213161}, end_lsn:{lsn:18544213283}, log_id:53509796, min_scn:{val:1723774794371366002, v:0}, max_scn:{val:1723774794371366002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544212758}, prev_proposal_id:7, committed_end_lsn:{lsn:18544213161}, data_checksum:1913811101, accum_checksum:3020376080, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774794558028, freeze_ts:1723774794558028, submit_ts:1723774794558035, flushed_ts:1723774794558733, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:698}, fs_cb_begin_ts=1723774794658128, log_life_time=100100) [2024-08-16 10:19:54.670612] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=797458432, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:54.670863] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash time detail, (compute_wash_size_time=115, refresh_score_time=65, wash_time=15) [2024-08-16 10:19:54.674040] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=16] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=3082786, clean_num=62914) [2024-08-16 10:19:54.695494] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.695520] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:19:54.695531] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.695542] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.695562] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.739321] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:54.739413] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=49, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:19:54.739447] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT APPEND COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:19:54.739459] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:54.739468] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:54.739473] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:54.740875] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:54.747137] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.747157] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:54.747168] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.747176] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.747190] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.754792] INFO [STORAGE.TRANS] rollback_tx (ob_tx_api.cpp:339) [2568][T1002_ArcSrv][T1001][YB42C0A80080-00061CCC9251D0C5-0-0] [lt=63] rollback tx(ret=0, *this={is_inited_:true, tenant_id_:1001, this:0x7fa6e0004030}, tx={this:0x7fa704c16b20, tx_id:{txid:283711663}, state:7, addr:"192.168.0.128:2882", tenant_id:1001, session_id:3225499190, assoc_session_id:3225499190, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:1723774794748492, op_sn:3, alloc_ts:1723774794749571, active_ts:1723774794749571, commit_ts:-1, finish_ts:1723774794753876, timeout_us:29999986, lock_timeout_us:-1, expire_ts:1723774824749557, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:1713182723, cluster_version:17180000768, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:-6002, commit_expire_ts:0, commit_task_.is_registered():false, ref:2}) [2024-08-16 10:19:54.755375] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=6] LogLoopThread round_cost_time(us)(round_cost_time=19) [2024-08-16 10:19:54.758111] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF49-0-0] [lt=48] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:19:54.758162] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF49-0-0] [lt=23] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:19:54.758202] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF49-0-0] [lt=39] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=90980, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:19:54.758213] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF49-0-0] [lt=9][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509797, log_task={header:{begin_lsn:{lsn:18544213283}, end_lsn:{lsn:18544213405}, log_id:53509797, min_scn:{val:1723774794371366003, v:0}, max_scn:{val:1723774794371366003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544213161}, prev_proposal_id:7, committed_end_lsn:{lsn:18544213283}, data_checksum:1913811101, accum_checksum:2775276534, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774794658104, freeze_ts:1723774794658104, submit_ts:1723774794658110, flushed_ts:1723774794658730, gen_to_freeze cost time:0, gen_to_submit cost time:6, submit_to_flush cost time:620}, fs_cb_begin_ts=1723774794758149, log_life_time=100045) [2024-08-16 10:19:54.772810] INFO [PALF] read (log_cache.cpp:100) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:54.795574] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.795608] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=30] handle task succ(id={id:1260}) [2024-08-16 10:19:54.795622] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.795632] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.795647] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.824647] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=975] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=8) [2024-08-16 10:19:54.839398] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:54.839509] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=9, avg_log_batch_cnt=1, total_group_log_size=1089, avg_group_log_size=121) [2024-08-16 10:19:54.839537] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:54.839551] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:54.839555] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:54.840190] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1089) [2024-08-16 10:19:54.840948] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:54.847208] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.847230] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:54.847244] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.847256] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.847276] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.858418] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF4F-0-0] [lt=31][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509798, log_task={header:{begin_lsn:{lsn:18544213405}, end_lsn:{lsn:18544213527}, log_id:53509798, min_scn:{val:1723774794371366004, v:0}, max_scn:{val:1723774794371366004, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544213283}, prev_proposal_id:7, committed_end_lsn:{lsn:18544213405}, data_checksum:1913811101, accum_checksum:4284610709, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774794758106, freeze_ts:1723774794758106, submit_ts:1723774794758136, flushed_ts:1723774794758681, gen_to_freeze cost time:0, gen_to_submit cost time:30, submit_to_flush cost time:545}, fs_cb_begin_ts=1723774794858408, log_life_time=100302) [2024-08-16 10:19:54.859103] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=34] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=847, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:19:54.871893] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [2782][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=7] ====== checkpoint timer task ====== [2024-08-16 10:19:54.871923] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [2782][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=24] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:19:54.872760] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA742F5-0-0] [lt=45] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=7, avg_log_submit_to_flush_time=590, avg_log_submit_to_slide_time=964) [2024-08-16 10:19:54.874428] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=797458432, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:54.874697] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash time detail, (compute_wash_size_time=113, refresh_score_time=84, wash_time=16) [2024-08-16 10:19:54.874750] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=3145700, clean_num=62914) [2024-08-16 10:19:54.895651] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.895674] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:19:54.895685] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.895696] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.895709] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.914863] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=29] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:19:54.931964] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80081-00061EFF86EC9418-0-0] [lt=26][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:54.936517] WDIAG [SQL.SESSION] find_sys_var_id_by_name (ob_system_variable_factory.cpp:901) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=7][errcode=-4182] sys var name not found(sys_var_name=profiling, lower_idx=176, ObSysVarFactory::ALL_SYS_VARS_COUNT=243, lbt()="0x12a0688c 0x101123db 0x5255ec7 0xe0bfc7c 0xe0c67ec 0xd4831ae 0xb634439 0x50f3e6e 0x5022bc9 0x4e90bea 0x4e8b77e 0x507dfb7 0x4e3c049 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:54.936545] WDIAG [SQL.SESSION] inner_get_sys_var (ob_basic_session_info.cpp:1200) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=22][errcode=-5044] fail to find sys var id by name(ret=-5044, sys_var_name=profiling, lbt()="0x12a0688c 0xe0bfe7f 0xe0bfd3e 0xe0c67ec 0xd4831ae 0xb634439 0x50f3e6e 0x5022bc9 0x4e90bea 0x4e8b77e 0x507dfb7 0x4e3c049 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:54.936554] WDIAG [SQL.SESSION] get_sys_variable_by_name (ob_basic_session_info.cpp:1921) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=8][errcode=-5044] fail to inner get sys var(var=profiling, ret=-5044) [2024-08-16 10:19:54.938424] WDIAG execute (ob_variable_set_executor.cpp:181) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=5][errcode=-5044] Unknown system variable 'profiling' [2024-08-16 10:19:54.938482] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=17][errcode=-5044] execute cmd failed(ret=-5044) [2024-08-16 10:19:54.938505] WDIAG [SQL] open (ob_result_set.cpp:157) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=21][errcode=-5044] execute plan failed(ret=-5044) [2024-08-16 10:19:54.938517] WDIAG [SERVER] response_result (ob_sync_cmd_driver.cpp:143) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=8][errcode=-5044] close result set fail(cret=-5044) [2024-08-16 10:19:54.938535] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=15][errcode=-5044] [RETRY] check if need retry(v={force_local_retry:false, stmt_retry_times:0, local_retry_times:0, err_:-5044, err_:"OB_ERR_SYS_VARIABLE_UNKNOWN", retry_type:0, client_ret:-5044}, need_retry=false) [2024-08-16 10:19:54.938556] WDIAG [SERVER] response_result (ob_sync_cmd_driver.cpp:149) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=20][errcode=-5044] result set open failed, check if need retry(ret=-5044, cli_ret=-5044, retry_ctrl_.need_retry()=0) [2024-08-16 10:19:54.938570] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:366) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=7] sending error packet(ob_error=-5044, client error=1193, extra_err_info=NULL, lbt()="0x12a0688c 0xa2161e0 0xa1c51c6 0x507e4c9 0x4e3c049 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:54.938615] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:549) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=13] dump txn free route audit_record(value=1048581, session->get_sessid()=3225589831, session->get_proxy_sessid()=13882346384134438935) [2024-08-16 10:19:54.938633] WDIAG [SERVER] do_process (obmp_query.cpp:823) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B7-0-0] [lt=14][errcode=-5044] execute query fail(ret=-5044, timeout_timestamp=1723784794936187) [2024-08-16 10:19:54.939485] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:54.939585] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1089) [2024-08-16 10:19:54.939613] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:54.939622] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:54.939628] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:54.941021] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:54.947280] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.947300] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:54.947310] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.947321] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.947335] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:54.949126] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1211) [2024-08-16 10:19:54.949298] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=8] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:54.949326] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=21] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:54.949453] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:19:54.949476] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=16][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:19:54.949489] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:19:54.949497] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=7][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:19:54.949581] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73503-0-0] [lt=20] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=1465, avg_log_submit_to_slide_time=1926) [2024-08-16 10:19:54.949610] INFO [PALF] read (log_cache.cpp:100) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:54.955105] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [31728][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=25] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:19:54.955175] INFO [STORAGE] update_info (ob_tenant_freeze_info_mgr.cpp:766) [31728][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=21] ObTenantFreezeInfoMgr success to update infos(snapshot_gc_ts=1723774792385636000, min_major_snapshot=1, cur_idx=1, info_list_[0]=[{freeze_version:1, schema_version:1, cluster_version:17180000768}, {freeze_version:1717696803757692000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1717783201511239000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1717869604194436000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1717956002182401000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1718042400038077000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1718128803012739000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1718215200877915000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718301603653574000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718388001794965000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718474404631237000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718560802392151000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718647200862409000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718733604116616000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718820002086291000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718906404976026000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718992802888678000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719079200791132000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719165603677720000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719252001650017000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719338404701723000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719424802791267000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719511200795513000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719597603653659000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719684001663096000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719770404568564000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719856802752420000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719943200797985000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1720029603818531000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1720116001847903000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1720202404863561000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1720288802835955000, schema_version:1718175945753488, cluster_version:17180000768}], info_list_[1]=[{freeze_version:1, schema_version:1, cluster_version:17180000768}, {freeze_version:1717696803757692000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1717783201511239000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1717869604194436000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1717956002182401000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1718042400038077000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1718128803012739000, schema_version:1714389289030344, cluster_version:17180000768}, {freeze_version:1718215200877915000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718301603653574000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718388001794965000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718474404631237000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718560802392151000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718647200862409000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718733604116616000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718820002086291000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718906404976026000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1718992802888678000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719079200791132000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719165603677720000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719252001650017000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719338404701723000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719424802791267000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719511200795513000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719597603653659000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719684001663096000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719770404568564000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719856802752420000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1719943200797985000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1720029603818531000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1720116001847903000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1720202404863561000, schema_version:1718175945753488, cluster_version:17180000768}, {freeze_version:1720288802835955000, schema_version:1718175945753488, cluster_version:17180000768}], snapshots_[0]=[], snapshots_[1]=[]) [2024-08-16 10:19:54.955574] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF_DUMP](palf_id=1262, self="192.168.0.128:2882", [SlidingWindow]={palf_id:1262, self:"192.168.0.128:2882", lsn_allocator:{max_log_id:34459293, max_lsn:8394837510, max_scn:{val:1723774794750151001, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:8394837510}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:8352894470}}, last_submit_lsn:{lsn:8394837388}, last_submit_end_lsn:{lsn:8394837510}, last_submit_log_id:34459293, last_submit_log_pid:2, max_flushed_lsn:{lsn:8394837388}, max_flushed_end_lsn:{lsn:8394837510}, max_flushed_log_pid:2, committed_end_lsn:{lsn:8394837510}, last_slide_log_id:34459293, last_slide_scn:{val:1723774794750151001, v:0}, last_slide_lsn:{lsn:8394837388}, last_slide_end_lsn:{lsn:8394837510}, last_slide_log_pid:2, last_slide_log_accum_checksum:1289736129, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x7fa6be52c530}, [StateMgr]={this:0x7fa6be5302b0, self:"192.168.0.128:2882", palf_id:1262, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:2}, leader:"192.168.0.133:2882", leader_epoch:6, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:34459294, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x7fa6be536b70, allow_vote:true}, [ConfigMgr]={palf_id:1262, self:"192.168.0.128:2882", alive_paxos_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744209401, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182753433, flag:0}, alive_paxos_replica_num:3, log_ms_meta:{version:3, proposal_id:2, prev:{version:1, config:{config_version:{proposal_id:2, config_seq:398}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744209401, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182753433, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:29241, lock_time:1723646524328842}}, curr:{version:1, config:{config_version:{proposal_id:2, config_seq:399}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744209401, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182753433, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29241, lock_time:1723646524499878}}, prev_log_proposal_id:2, prev_lsn:{lsn:7750282582}, prev_mode_pid:2}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:2, config_seq:399}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1723774794248824, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1723774794248824, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x7fa6be52dc70}, [ModeMgr]={palf_id:1262, self:"192.168.0.128:2882", applied_mode_meta:{version:1, proposal_id:2, mode_version:0, access_mode:1, ref_scn:{val:1720516541224759000, v:0}}, accepted_mode_meta:{version:1, proposal_id:2, mode_version:0, access_mode:1, ref_scn:{val:1720516541224759000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:2, mode_version:0, access_mode:1, ref_scn:{val:1720516541224759000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1262, is_inited:true, min_block_max_scn:{val:1721957131723543002, v:0}, min_block_id:59, min_block_min_scn:{val:1721929793432294003, v:0}, base_lsn_for_block_gc:{lsn:8253886464}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:2}, log_config_meta:{version:3, proposal_id:2, prev:{version:1, config:{config_version:{proposal_id:2, config_seq:398}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744209401, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182753433, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:29241, lock_time:1723646524328842}}, curr:{version:1, config:{config_version:{proposal_id:2, config_seq:399}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744209401, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182753433, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29241, lock_time:1723646524499878}}, prev_log_proposal_id:2, prev_lsn:{lsn:7750282582}, prev_mode_pid:2}, log_snapshot_meta:{version:1, base_lsn:{lsn:8253886464}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:2, mode_version:0, access_mode:1, ref_scn:{val:1720516541224759000, v:0}}}, log_meta_storage:{log_tail:{lsn:1048576}, readable_log_tail:{lsn:1048576}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:8455200768}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1262}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1262/meta", dir_fd:202, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:66056192, block_header_serialize_buf_:0x7fa6be532438, flashback_version:0}, log_storage:{log_tail:{lsn:8394837510}, readable_log_tail:{lsn:8394837510}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:8388096000}, min_scn:{val:1723770206317513001, v:0}, curr_block_id:125, palf_id:1262}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1262/log", dir_fd:203, min_block_id:59, max_block_id:126, curr_writable_block_id:125}, logical_block_size_:67104768, curr_block_writable_size_:60363258, block_header_serialize_buf_:0x7fa6be5340b8, flashback_version:0}, palf_epoch:4, last_purge_throttling_ts:1723516779808590, this:0x7fa6be5309f0}, [Reconfirm]={palf_id:1262, self:"192.168.0.128:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x7fa6be530430}) [2024-08-16 10:19:54.955764] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=181][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1262, self="192.168.0.128:2882", tmp_server={server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}) [2024-08-16 10:19:54.955776] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF_DUMP](palf_id=1262, self="192.168.0.128:2882", ack_info_list=[]) [2024-08-16 10:19:54.955784] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1262, self="192.168.0.128:2882", tmp_server={server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}) [2024-08-16 10:19:54.955792] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=7] current log_task status(palf_id=1262, self="192.168.0.128:2882", log_id=34459294, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[]) [2024-08-16 10:19:54.959158] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT FLUSH CB COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=37, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:19:54.968923] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B8-0-0] [lt=7] succeed to calculate base table sharding info(table_id=18446744073709551614, ref_table_id=12008, is_global_index=false) [2024-08-16 10:19:54.968962] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B8-0-0] [lt=22] total rowcount, use statistics(table_meta_info_.table_row_count_=2, table_meta_info_.average_row_size_=4.000000000000000000e+01, table_meta_info_.micro_block_count_=0, table_meta_info_.part_size_=8.000000000000000000e+01) [2024-08-16 10:19:54.969621] WDIAG [SQL.PC] common_free (ob_lib_cache_object_manager.cpp:141) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B8-0-0] [lt=13][errcode=0] set logical del time(cache_obj->get_logical_del_time()=10451107228542, cache_obj->added_lc()=false, cache_obj->get_object_id()=254219, cache_obj->get_tenant_id()=1002, lbt()="0x12a0688c 0xaa7c9e4 0x4e44d8a 0x4e36df8 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:54.984770] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B9-0-0] [lt=23] succeed to calculate base table sharding info(table_id=18446744073709551614, ref_table_id=12008, is_global_index=false) [2024-08-16 10:19:54.984806] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B9-0-0] [lt=23] total rowcount, use statistics(table_meta_info_.table_row_count_=2, table_meta_info_.average_row_size_=4.000000000000000000e+01, table_meta_info_.micro_block_count_=0, table_meta_info_.part_size_=8.000000000000000000e+01) [2024-08-16 10:19:54.985250] WDIAG [SQL.PC] common_free (ob_lib_cache_object_manager.cpp:141) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1B9-0-0] [lt=14][errcode=0] set logical del time(cache_obj->get_logical_del_time()=10451107244172, cache_obj->added_lc()=false, cache_obj->get_object_id()=254220, cache_obj->get_tenant_id()=1002, lbt()="0x12a0688c 0xaa7c9e4 0x4e44d8a 0x4e36df8 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:54.986658] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [2566][T1002_ArcSender][T1002][YB42C0A80080-00061CCC9231D0C5-0-0] [lt=21][errcode=-4018] try free send task failed(ret=-4018) [2024-08-16 10:19:54.995725] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:54.995747] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:54.995757] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:54.995769] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:54.995790] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.001659] WDIAG [SQL.RESV] find_column_in_joined_table (ob_column_namespace_checker.cpp:482) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=22][errcode=-5207] column in joined table is ambiguous(q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:55.001803] WDIAG [SQL.RESV] find_column_in_joined_table (ob_column_namespace_checker.cpp:468) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=142][errcode=-5207] find column in left table failed(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:55.001838] WDIAG [SQL.RESV] find_column_in_joined_table (ob_column_namespace_checker.cpp:468) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=34][errcode=-5207] find column in left table failed(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:55.001864] WDIAG [SQL.RESV] find_column_in_joined_table (ob_column_namespace_checker.cpp:468) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=25][errcode=-5207] find column in left table failed(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:55.001883] WDIAG [SQL.RESV] find_column_in_joined_table (ob_column_namespace_checker.cpp:468) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=19][errcode=-5207] find column in left table failed(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:55.001904] WDIAG [SQL.RESV] check_table_column_namespace (ob_column_namespace_checker.cpp:103) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=20][errcode=-5207] find column in table failed(ret=-5207) [2024-08-16 10:19:55.001911] WDIAG [SQL.RESV] resolve_table_column_expr (ob_dml_resolver.cpp:3115) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=6][errcode=-5207] column not found in table(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}) [2024-08-16 10:19:55.001985] WDIAG [SQL.RESV] resolve_table_column_ref (ob_select_resolver.cpp:5023) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=24][errcode=-5207] resolve table column expr failed(ret=-5207, q_name={database_name:"", table_name:"", column:"id", dblink_name:"", is_star:false, ref_expr:{item_type:"T_REF_COLUMN", result_type:{meta:{type:"", collation:"binary", coercibility:"NUMERIC"}, accuracy:{length:-1, precision:-1, scale:-1}, flag:0, calc_type:{type:"NULL", collation:"invalid_type", coercibility:"INVALID"}}, expr_info:["IS_COLUMN", "CNT_COLUMN"], rel_id:[], table_id:18446744073709551615, column_id:18446744073709551615, database_name:"", table_name:"", synonym_name:"", synonym_db_name:"", column_name:"", column_flags:0, enum_set_values:[], is_lob_column:false, is_joined_dup_column:false, is_unpivot_mocked_column:false, is_hidden:false, from_alias_table:false, is_rowkey_column:false, is_unique_key_column:false, is_mul_key_column:false, is_strict_json_column:0, srs_id:18446744073709551615, udt_set_id:0}, parents_expr_info:[], parent_aggr_level:-1, access_idents:[{access_name:"id", access_index:-1, type:-1, params:[]}], current_resolve_level:0, is_access_root:true}, lbt()="0x12a0688c 0xd8c56b9 0xd8c5596 0xd8c3ac9 0xd8c48ec 0xd6fefc2 0xd6f888d 0xd683d91 0xd8b7a20 0xd889202 0xd898997 0xd7c0576 0xd73a576 0xd739d50 0xd6e5e94 0xd884fec 0xd898997 0xcc3a2d5 0x51eb425 0x504b6d2 0x5044bba 0x5042480 0x4e55d67 0x4e3568a 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:55.007666] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=51] succeed to calculate base table sharding info(table_id=18446744073709551609, ref_table_id=516604, is_global_index=false) [2024-08-16 10:19:55.007708] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=27] total rowcount, use statistics(table_meta_info_.table_row_count_=806, table_meta_info_.average_row_size_=9.139000000000000000e+03, table_meta_info_.micro_block_count_=40, table_meta_info_.part_size_=7.366034000000000000e+06) [2024-08-16 10:19:55.011687] INFO [CLOG] update_upstream (ob_remote_location_adaptor.cpp:91) [2533][T1002_LogRessvr][T1002][YB42C0A80080-00061CCC91B1D0C5-0-0] [lt=16] primary tenant, just skip [2024-08-16 10:19:55.011807] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=16] gc stale ls task succ [2024-08-16 10:19:55.011945] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=5] gc stale ls task succ [2024-08-16 10:19:55.011977] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=20] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:19:55.012008] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=22] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:19:55.012042] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=32] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:19:55.012988] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=16] succeed to calculate base table sharding info(table_id=18446744073709551608, ref_table_id=525321, is_global_index=false) [2024-08-16 10:19:55.013030] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=29] total rowcount, use statistics(table_meta_info_.table_row_count_=12926, table_meta_info_.average_row_size_=7.960000000000000000e+02, table_meta_info_.micro_block_count_=55, table_meta_info_.part_size_=1.028909600000000000e+07) [2024-08-16 10:19:55.015567] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=18] succeed to calculate base table sharding info(table_id=18446744073709551607, ref_table_id=559954, is_global_index=false) [2024-08-16 10:19:55.015602] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=25] total rowcount, use statistics(table_meta_info_.table_row_count_=768506, table_meta_info_.average_row_size_=1.628000000000000000e+03, table_meta_info_.micro_block_count_=3774, table_meta_info_.part_size_=1.251127768000000000e+09) [2024-08-16 10:19:55.016150] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=15] succeed to calculate base table sharding info(table_id=18446744073709551606, ref_table_id=545627, is_global_index=false) [2024-08-16 10:19:55.016183] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=24] total rowcount, use statistics(table_meta_info_.table_row_count_=140460, table_meta_info_.average_row_size_=2.716000000000000000e+03, table_meta_info_.micro_block_count_=6132, table_meta_info_.part_size_=3.814893600000000000e+08) [2024-08-16 10:19:55.016640] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=12] succeed to calculate base table sharding info(table_id=18446744073709551605, ref_table_id=583346, is_global_index=false) [2024-08-16 10:19:55.016669] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=20] total rowcount, use statistics(table_meta_info_.table_row_count_=144, table_meta_info_.average_row_size_=1.800000000000000000e+02, table_meta_info_.micro_block_count_=1, table_meta_info_.part_size_=2.592000000000000000e+04) [2024-08-16 10:19:55.016741] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=32] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:55.016742] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:55.016912] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=13] succeed to calculate base table sharding info(table_id=18446744073709551604, ref_table_id=500059, is_global_index=false) [2024-08-16 10:19:55.016940] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=20] total rowcount, use statistics(table_meta_info_.table_row_count_=3519, table_meta_info_.average_row_size_=1.380000000000000000e+02, table_meta_info_.micro_block_count_=3, table_meta_info_.part_size_=4.856220000000000000e+05) [2024-08-16 10:19:55.018850] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=22] PNIO [ratelimit] time: 1723774795018848, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007631, add_bytes: 0 [2024-08-16 10:19:55.021403] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2BFE-0-0] [lt=21] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774795021360) [2024-08-16 10:19:55.024778] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=30] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=7) [2024-08-16 10:19:55.034446] INFO [SQL.PC] inner_add_cache_obj (ob_pcv_set.cpp:266) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=16] has identical pcv(is_same=true, pcv=0x7fa704562ae0) [2024-08-16 10:19:55.035574] WDIAG [SQL.ENG] set_sqcs_accessed_location (ob_px_util.cpp:1145) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=22][errcode=-4016] we do not find this addr's execution sqc(ret=-4016, n_locations=0, locations.size()=1, sqcs=[{need_report:false, execution_id:338495225, qc_id:1, sqc_id:0, dfo_id:1, exec_addr:"192.168.0.124:2882", qc_addr:"192.168.0.128:2882", qc_ch_info:{chid:0, type:0, peer:"0.0.0.0:0", role:0, tenant_id:0, state_:0}, sqc_ch_info:{chid:0, type:0, peer:"0.0.0.0:0", role:0, tenant_id:0, state_:0}, task_count:0, max_task_count:1, min_task_count:1, thread_inited:false, thread_finish:false, px_int_id:{query_interrupt_id:task(1720514724410946,68719477589), px_interrupt_id:task(1720514724410946,68723676004)}, transmit_use_interm_result:false, recieve_use_interm_result:false, serial_receive_channels:[], sqc_order_gi_tasks:false}], locations=[{tablet_id:{id:1152921504606866871}, ls_id:{id:1260}, server:"192.168.0.128:2882", in_retry:0, partition_id:524323, first_level_part_id:18446744073709551615}]) [2024-08-16 10:19:55.035625] WDIAG [SQL.ENG] set_dfo_accessed_location (ob_px_util.cpp:1050) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=48][errcode=-4016] failed to set sqc accessed location(ret=-4016, table_location_key=18446744073709551609, ref_table_id=524299, table_loc={loc_meta:{table_loc_id:18446744073709551609, ref_table_id:524299, related_table_ids:[516604], use_dist_das:0, select_leader:1, is_dup_table:0, is_weak_read:0, unuse_related_pruning:0, is_external_table:0}, tablet_locs:[{tablet_id:{id:1152921504606866871}, ls_id:{id:1260}, server:"192.168.0.128:2882", in_retry:0, partition_id:524323, first_level_part_id:18446744073709551615}], is_writing:0, is_reading:0, need_refresh:0, is_fk_check:0}) [2024-08-16 10:19:55.035655] WDIAG [SQL.ENG] alloc_by_data_distribution_inner (ob_px_util.cpp:382) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=28][errcode=-4016] fail to set all table partition for tsc(ret=-4016, scan_ops.count()=3, dml_op=NULL, table_location_key=18446744073709551604, ref_table_id=500061, locations=[{tablet_id:{id:1152921504606847007}, ls_id:{id:1024}, server:"192.168.0.124:2882", in_retry:0, partition_id:500061, first_level_part_id:18446744073709551615}]) [2024-08-16 10:19:55.035673] WDIAG [SQL.ENG] alloc_by_data_distribution (ob_px_util.cpp:63) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=16][errcode=-4016] failed to alloc data distribution(ret=-4016) [2024-08-16 10:19:55.035680] WDIAG [SQL.ENG] init_all_dfo_channel (ob_dfo_scheduler.cpp:321) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=7][errcode=-4016] fail to alloc data distribution(ret=-4016) [2024-08-16 10:19:55.035687] WDIAG [SQL.ENG] startup_msg_loop (ob_px_scheduler.cpp:128) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=6][errcode=-4016] fail to init all dfo channel(ret=-4016) [2024-08-16 10:19:55.035697] WDIAG [SQL.ENG] fetch_rows (ob_px_fifo_coord_op.cpp:135) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=9][errcode=-4016] initial dfos NOT dispatched successfully(ret=-4016) [2024-08-16 10:19:55.035704] WDIAG [SQL.ENG] fetch_rows (ob_px_fifo_coord_op.cpp:244) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=4][errcode=-4016] QC get error code(ret=-4016, ret_terminate=0) [2024-08-16 10:19:55.035715] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=9][errcode=-4016] get next batch failed(ret=-4016, eval_ctx={batch_idx:0, batch_size:256, max_batch_size:256, frames_:0x7fa710a0efa0}, id=0, op_name="PHY_PX_FIFO_COORD") [2024-08-16 10:19:55.035731] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1499) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=15][errcode=-4016] get next batch failed(ret=-4016) [2024-08-16 10:19:55.035737] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=5][errcode=-4016] get next row from exec result failed(ret=-4016) [2024-08-16 10:19:55.035743] WDIAG [SERVER] response_query_result (ob_query_driver.cpp:299) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=5][errcode=-4016] fail to iterate and response(ret=-4016, row_num=0, can_retry=true) [2024-08-16 10:19:55.035751] WDIAG [SERVER] response_result (ob_sync_plan_driver.cpp:101) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=7][errcode=-4016] response query result fail(ret=-4016) [2024-08-16 10:19:55.035757] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=3][errcode=-4016] [RETRY] check if need retry(v={force_local_retry:false, stmt_retry_times:0, local_retry_times:0, err_:-4016, err_:"OB_ERR_UNEXPECTED", retry_type:0, client_ret:-4016}, need_retry=false) [2024-08-16 10:19:55.035772] WDIAG [SERVER] response_result (ob_sync_plan_driver.cpp:113) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=14][errcode=-4016] result response failed, check if need retry(ret=-4016, cli_ret=-4016, retry_ctrl_.need_retry()=0) [2024-08-16 10:19:55.035801] WDIAG [SQL.ENG] destroy_all_channel (ob_px_coord_op.cpp:672) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=7][errcode=-4201] fail unlink channel(qc_ci={chid:0, type:0, peer:"0.0.0.0:0", role:0, tenant_id:0, state_:0}, ret=-4201) [2024-08-16 10:19:55.035817] WDIAG [SQL.ENG] destroy_all_channel (ob_px_coord_op.cpp:672) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=15][errcode=-4201] fail unlink channel(qc_ci={chid:0, type:0, peer:"0.0.0.0:0", role:0, tenant_id:0, state_:0}, ret=-4201) [2024-08-16 10:19:55.035831] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=7] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x7fa7311c74a0, tx_id:{txid:0}, state:1, addr:"192.168.0.128:2882", tenant_id:1002, session_id:3225589831, assoc_session_id:3225589831, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:-1, tx_consistency_type:0, isolation:-1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1723774794985107, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:true, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:-1, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x7fa66c6a21f8, id:3225589831, deser:false, tenant:"epean", tenant_id:1002, effective_tenant:"epean", effective_tenant_id:1002, database:"ep_trade_test", user:"epean_trade_test@%", consistency_level:3, session_state:2, autocommit:true, tx:0x7fa7311c74a0}, exec_ctx.get_errcode()=-4016) [2024-08-16 10:19:55.035975] WDIAG [SERVER] response_result (ob_sync_plan_driver.cpp:120) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=56][errcode=-4016] close result set fail(cret=-4016) [2024-08-16 10:19:55.036000] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:366) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=16] sending error packet(ob_error=-4016, client error=-4016, extra_err_info=NULL, lbt()="0x12a0688c 0xa2161e0 0xa1c51c6 0x4e7e290 0x4e35890 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:55.036046] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:549) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=10] dump txn free route audit_record(value=1048581, session->get_sessid()=3225589831, session->get_proxy_sessid()=13882346384134438935) [2024-08-16 10:19:55.036064] WDIAG [SERVER] do_process (obmp_query.cpp:823) [11511][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC39FE1BA-0-0] [lt=14][errcode=-4016] execute query fail(ret=-4016, timeout_timestamp=1723784794999607) [2024-08-16 10:19:55.039552] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:55.039654] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:55.039668] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:55.039676] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:55.040132] INFO [STORAGE.DUP_TABLE] ls_loop_handle (ob_dup_table_util.cpp:624) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=29] [DUP_TABLE_DIAG: Dup Interface Stat] tenant: 1002, ls: 1079, is_master: False, {dup_table_follower_read_succ_cnt_:780757, dup_table_follower_read_tablet_not_exist_cnt_:0, dup_table_follower_read_tablet_not_ready_cnt_:0, dup_table_follower_read_lease_expired_cnt_:71, dup_table_redo_sync_succ_cnt_:0, dup_table_redo_sync_fail_cnt_:0, dup_table_max_applying_scn_:{val:1723774794371366001, v:0}, dup_table_log_entry_cnt_:2761305, dup_table_log_entry_total_size_:1070041539, dup_table_log_replay_total_time_:0, dup_table_log_deser_total_time_:0, dup_table_lease_log_sync_total_time_:4042970, dup_table_tablet_log_sync_total_time_:2501510} [2024-08-16 10:19:55.040166] INFO [STORAGE.DUP_TABLE] print_lease_diag_info_log (ob_dup_table_lease.cpp:658) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=16] [DUP_TABLE_DIAG: Follower Lease Info] tenant: 1002, ls: 1079 , current_time = 2024-08-16 10:19:55.40149 [DUP_TABLE_DIAG: Follower Lease] is_expired=False, request_ts=1723774792938530, request_ts(date)=2024-08-16 10:19:52.938530, lease_expired_time=2024-08-16 10:19:58.938530, lease_interval_us=6000000, last_lease_scn={val:1723774792869514004, v:0}, lease_acquire_scn={val:1722500589629865001, v:0} [2024-08-16 10:19:55.040235] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=31] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1089) [2024-08-16 10:19:55.041096] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=18][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:55.047364] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.047386] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:55.047396] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.047405] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.047419] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.047892] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2417][T1002_Occam][T1002][YB42C0A80080-00061CCCBF7D385C-0-0] [lt=21] for each ls(succ_num=4, ret=0, ret="OB_SUCCESS") [2024-08-16 10:19:55.048147] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=35] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:19:55.058515] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF5B-0-0] [lt=33][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509800, log_task={header:{begin_lsn:{lsn:18544213649}, end_lsn:{lsn:18544213771}, log_id:53509800, min_scn:{val:1723774794872386000, v:0}, max_scn:{val:1723774794872386000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544213527}, prev_proposal_id:7, committed_end_lsn:{lsn:18544213649}, data_checksum:1913811101, accum_checksum:929496209, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774794958461, freeze_ts:1723774794958461, submit_ts:1723774794958466, flushed_ts:1723774794959094, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:628}, fs_cb_begin_ts=1723774795058505, log_life_time=100044) [2024-08-16 10:19:55.058623] INFO [LIB] stat (utility.h:1197) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=36] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=39, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:19:55.058657] INFO [LIB] stat (utility.h:1197) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=32] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=5, this=0x7fa713668b28, extra_info="") [2024-08-16 10:19:55.064488] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=33][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:19:55.064508] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:19:55.070431] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AA8-0-0] [lt=21] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:19:55.070466] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AA8-0-0] [lt=21] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774794750151002, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774794750151002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:19:55.071957] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1723774795071955, bytes: 1496435596499, bw: 0.133034 MB/s, add_ts: 1000748, add_bytes: 139601 [2024-08-16 10:19:55.071988] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8588-0-0] [lt=40] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:19:55.072026] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8588-0-0] [lt=13] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1211, avg_group_log_size=121) [2024-08-16 10:19:55.072038] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8588-0-0] [lt=11] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1211) [2024-08-16 10:19:55.075219] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=799555584, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:55.075477] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=32] Wash time detail, (compute_wash_size_time=131, refresh_score_time=66, wash_time=16) [2024-08-16 10:19:55.078760] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=15] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=0, clean_num=62914) [2024-08-16 10:19:55.095657] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:19:55.095808] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.095829] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:55.095838] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.095854] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.095869] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.105477] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [2783][T1001_Flush][T1001][Y0-0000000000000000-0-0] [lt=8] ====== traversal_flush timer task ====== [2024-08-16 10:19:55.105514] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:338) [2783][T1001_Flush][T1001][Y0-0000000000000000-0-0] [lt=29] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:19:55.112293] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7D5-0-0] [lt=86] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:19:55.121520] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2659][T1001_Occam][T1001][YB42C0A80080-00061CCCBF8D385C-0-0] [lt=22] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:19:55.137910] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [2774][T1001_LCLSender][T1001][Y0-0000000000000000-0-0] [lt=34] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=1135, total_destructed_detector=1135, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x7fa704be42b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-08-16 10:19:55.139637] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:55.139769] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:55.139793] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:55.139802] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:55.140471] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=631, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:19:55.140509] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=28, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:19:55.140535] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:19:55.141166] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:120) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=18] tenant info is invalid(ret=-4076, query_ts=0, standby_scn={val:0, v:0}, this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:19:55.141195] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=27][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:55.147443] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.147465] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:55.147475] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.147484] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.147501] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.148750] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A4D-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1211, avg_group_log_size=121) [2024-08-16 10:19:55.148781] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A4D-0-0] [lt=28] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1211) [2024-08-16 10:19:55.151135] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=22] current clock is (clock_=0, max_seq_=0, mem_limit=1643328294, cur_mem_hold=289406976, attr_.tenant_id_=1002) [2024-08-16 10:19:55.160535] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAD2-0-0] [lt=26][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=8, len=115, pos=86, bitmap=NULL) [2024-08-16 10:19:55.161562] INFO [CLOG] is_in_sync (ob_log_handler.cpp:546) [21123][T1_L0_G0][T1002][YB42C0A8007F-000616D421E91A0B-0-0] [lt=21] is_in_sync(ret=0, id=1079, is_log_sync=true, is_need_rebuild=false) [2024-08-16 10:19:55.161639] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [21123][T1_L0_G0][T1][YB42C0A8007F-000616D421E91A0B-0-0] [lt=22][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:55.163332] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF61-0-0] [lt=14][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509801, log_task={header:{begin_lsn:{lsn:18544213771}, end_lsn:{lsn:18544213893}, log_id:53509801, min_scn:{val:1723774794872386001, v:0}, max_scn:{val:1723774794872386001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544213649}, prev_proposal_id:7, committed_end_lsn:{lsn:18544213771}, data_checksum:1913811101, accum_checksum:1804157276, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774795058487, freeze_ts:1723774795058487, submit_ts:1723774795058491, flushed_ts:1723774795059107, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:616}, fs_cb_begin_ts=1723774795163321, log_life_time=104834) [2024-08-16 10:19:55.183061] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=19] gc stale ls task succ [2024-08-16 10:19:55.185180] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE7A-0-0] [lt=25] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:19:55.185234] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE7A-0-0] [lt=33] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774795147916000, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774795147916000, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:19:55.191206] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2520][T1002_TenantMet][T1002][Y0-0000000000000000-0-0] [lt=70] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:17374, free_obj_hold_cnt:593, allocator used:71436792, allocator total:73453184}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:1994, free_obj_hold_cnt:6, allocator used:131182000, allocator total:131182000}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():63488, full allocator used:1600, full allocator total:129024}, tablets_mem=204764208, tablets_mem_limit=1288490180, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:7, allocator used:17024, allocator total:65408}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:106, free_obj_hold_cnt:5057, allocator used:10243392, allocator total:10596096}, pending_cnt=0, wait_gc_count=0, tablet count=19368) [2024-08-16 10:19:55.195893] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.195921] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=26] handle task succ(id={id:1260}) [2024-08-16 10:19:55.195930] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.195940] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.195954] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.214727] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=20] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574604, offset=42138437}, end_cursor:ObLogCursor{file_id=65, log_id=18574605, offset=42138678}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516c4050, len:1211, log_data_len:241, seq:18574604, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:55.215159] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=31] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574605, offset=42138678}, end_cursor:ObLogCursor{file_id=65, log_id=18574606, offset=42138927}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751694050, len:970, log_data_len:249, seq:18574605, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:55.216692] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BC9D-0-0] [lt=28] push gts task success(*task={:""}) [2024-08-16 10:19:55.224877] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=19] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=6) [2024-08-16 10:19:55.225589] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BC9D-0-0] [lt=19] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:19:55.228001] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=20] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574606, offset=42138927}, end_cursor:ObLogCursor{file_id=65, log_id=18574607, offset=42139176}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7519c0050, len:721, log_data_len:249, seq:18574606, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:55.228577] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=24] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574607, offset=42139176}, end_cursor:ObLogCursor{file_id=65, log_id=18574608, offset=42139425}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751648050, len:472, log_data_len:249, seq:18574607, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:55.235995] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=19] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:55.236346] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:55.236419] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:55.236813] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=14] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:55.236932] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:55.237273] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=23] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:55.237374] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:55.239744] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:55.239887] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=36] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:55.239902] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:55.239910] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:55.241245] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:55.241366] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7350B-0-0] [lt=37] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:19:55.241387] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7350B-0-0] [lt=20] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=1246, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:19:55.247530] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.247551] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:55.247561] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.247571] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.247586] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.258809] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF67-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1614, avg_group_log_size=146) [2024-08-16 10:19:55.270164] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=18] current monitor number(seq_=-1) [2024-08-16 10:19:55.272745] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=521, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:19:55.272780] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1658) [2024-08-16 10:19:55.272801] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=36, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:19:55.272842] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=28, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:19:55.273198] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74301-0-0] [lt=39] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:19:55.273213] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74301-0-0] [lt=14] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=994, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:19:55.279408] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=799555584, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:55.279723] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=41] Wash time detail, (compute_wash_size_time=178, refresh_score_time=76, wash_time=25) [2024-08-16 10:19:55.282879] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=62914, clean_num=62914) [2024-08-16 10:19:55.295969] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.295991] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:55.296002] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.296011] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.296024] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.298861] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=16] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:19:55.298883] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=20] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=4933, mem_used=39515136) [2024-08-16 10:19:55.302136] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=85] get all snapshots(ret=0, snapshots.count()=192, snapshots=[{snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213191, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213192, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213193, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213194, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213195, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213196, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213197, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213198, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213199, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213200, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213201, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213202, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213203, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213204, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213205, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213206, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213207, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213208, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213209, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213210, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213211, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213212, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213213, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213214, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213215, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213216, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213217, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213218, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213219, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213220, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213221, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213222, comment:null}]) [2024-08-16 10:19:55.302288] INFO [STORAGE] update_info (ob_tenant_freeze_info_mgr.cpp:766) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=84] ObTenantFreezeInfoMgr success to update infos(snapshot_gc_ts=1723774790629965000, min_major_snapshot=1, cur_idx=1, info_list_[0]=[{freeze_version:1, schema_version:1, cluster_version:17180000768}, {freeze_version:1717675200551380002, schema_version:1717665507792856, cluster_version:17180000768}, {freeze_version:1717761601458995000, schema_version:1717756836982992, cluster_version:17180000768}, {freeze_version:1717848002963687000, schema_version:1717756836982992, cluster_version:17180000768}, {freeze_version:1717934404565116000, schema_version:1717756836982992, cluster_version:17180000768}, {freeze_version:1718020800815076000, schema_version:1717756836982992, cluster_version:17180000768}, {freeze_version:1718107202784772000, schema_version:1718100103349688, cluster_version:17180000768}, {freeze_version:1718193603756584000, schema_version:1718183629728720, cluster_version:17180000768}, {freeze_version:1718280004757524000, schema_version:1718252152287688, cluster_version:17180000768}, {freeze_version:1718366401110773000, schema_version:1718358901643192, cluster_version:17180000768}, {freeze_version:1718452801308324000, schema_version:1718358901643192, cluster_version:17180000768}, {freeze_version:1718539204596315000, schema_version:1718358901643192, cluster_version:17180000768}, {freeze_version:1718625601620260000, schema_version:1718616144307376, cluster_version:17180000768}, {freeze_version:1718712001011391000, schema_version:1718703000567536, cluster_version:17180000768}, {freeze_version:1718798402220563000, schema_version:1718789800578568, cluster_version:17180000768}, {freeze_version:1718884803704209000, schema_version:1718877149331840, cluster_version:17180000768}, {freeze_version:1718971200257598000, schema_version:1718965154357056, cluster_version:17180000768}, {freeze_version:1719057601200611000, schema_version:1718996463302080, cluster_version:17180000768}, {freeze_version:1719144004280300000, schema_version:1718996463302080, cluster_version:17180000768}, {freeze_version:1719230400900297000, schema_version:1719223218688584, cluster_version:17180000768}, {freeze_version:1719316803672489000, schema_version:1719309093487688, cluster_version:17180000768}, {freeze_version:1719403200851192000, schema_version:1719395001003776, cluster_version:17180000768}, {freeze_version:1719489602953423000, schema_version:1719481664171696, cluster_version:17180000768}, {freeze_version:1719576001720242000, schema_version:1719569884720288, cluster_version:17180000768}, {freeze_version:1719662403576454000, schema_version:1719630973004440, cluster_version:17180000768}, {freeze_version:1719748802348070000, schema_version:1719630973004440, cluster_version:17180000768}, {freeze_version:1719835204977403000, schema_version:1719822612496560, cluster_version:17180000768}, {freeze_version:1719921603646937000, schema_version:1719913115910976, cluster_version:17180000768}, {freeze_version:1720008002818629000, schema_version:1719994142500984, cluster_version:17180000768}, {freeze_version:1720094402994133000, schema_version:1720088798802896, cluster_version:17180000768}, {freeze_version:1720180802838815000, schema_version:1720173752016384, cluster_version:17180000768}, {freeze_version:1720267200219042000, schema_version:1720173752016384, cluster_version:17180000768}], info_list_[1]=[{freeze_version:1, schema_version:1, cluster_version:17180000768}, {freeze_version:1717675200551380002, schema_version:1717665507792856, cluster_version:17180000768}, {freeze_version:1717761601458995000, schema_version:1717756836982992, cluster_version:17180000768}, {freeze_version:1717848002963687000, schema_version:1717756836982992, cluster_version:17180000768}, {freeze_version:1717934404565116000, schema_version:1717756836982992, cluster_version:17180000768}, {freeze_version:1718020800815076000, schema_version:1717756836982992, cluster_version:17180000768}, {freeze_version:1718107202784772000, schema_version:1718100103349688, cluster_version:17180000768}, {freeze_version:1718193603756584000, schema_version:1718183629728720, cluster_version:17180000768}, {freeze_version:1718280004757524000, schema_version:1718252152287688, cluster_version:17180000768}, {freeze_version:1718366401110773000, schema_version:1718358901643192, cluster_version:17180000768}, {freeze_version:1718452801308324000, schema_version:1718358901643192, cluster_version:17180000768}, {freeze_version:1718539204596315000, schema_version:1718358901643192, cluster_version:17180000768}, {freeze_version:1718625601620260000, schema_version:1718616144307376, cluster_version:17180000768}, {freeze_version:1718712001011391000, schema_version:1718703000567536, cluster_version:17180000768}, {freeze_version:1718798402220563000, schema_version:1718789800578568, cluster_version:17180000768}, {freeze_version:1718884803704209000, schema_version:1718877149331840, cluster_version:17180000768}, {freeze_version:1718971200257598000, schema_version:1718965154357056, cluster_version:17180000768}, {freeze_version:1719057601200611000, schema_version:1718996463302080, cluster_version:17180000768}, {freeze_version:1719144004280300000, schema_version:1718996463302080, cluster_version:17180000768}, {freeze_version:1719230400900297000, schema_version:1719223218688584, cluster_version:17180000768}, {freeze_version:1719316803672489000, schema_version:1719309093487688, cluster_version:17180000768}, {freeze_version:1719403200851192000, schema_version:1719395001003776, cluster_version:17180000768}, {freeze_version:1719489602953423000, schema_version:1719481664171696, cluster_version:17180000768}, {freeze_version:1719576001720242000, schema_version:1719569884720288, cluster_version:17180000768}, {freeze_version:1719662403576454000, schema_version:1719630973004440, cluster_version:17180000768}, {freeze_version:1719748802348070000, schema_version:1719630973004440, cluster_version:17180000768}, {freeze_version:1719835204977403000, schema_version:1719822612496560, cluster_version:17180000768}, {freeze_version:1719921603646937000, schema_version:1719913115910976, cluster_version:17180000768}, {freeze_version:1720008002818629000, schema_version:1719994142500984, cluster_version:17180000768}, {freeze_version:1720094402994133000, schema_version:1720088798802896, cluster_version:17180000768}, {freeze_version:1720180802838815000, schema_version:1720173752016384, cluster_version:17180000768}, {freeze_version:1720267200219042000, schema_version:1720173752016384, cluster_version:17180000768}], snapshots_[0]=[{snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213191, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213192, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213193, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213194, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213195, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213196, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213197, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213198, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213199, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213200, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213201, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213202, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213203, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213204, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213205, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213206, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213207, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213208, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213209, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213210, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213211, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213212, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213213, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213214, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213215, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213216, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213217, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213218, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213219, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213220, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213221, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213222, comment:null}], snapshots_[1]=[{snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213191, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213192, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213193, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213194, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213195, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213196, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213197, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213198, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213199, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213200, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213201, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213202, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213203, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213204, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213205, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213206, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213207, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213208, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213209, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213210, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213211, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213212, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213213, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213214, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213215, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213216, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213217, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213218, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213219, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213220, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213221, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213222, comment:null}]) [2024-08-16 10:19:55.339806] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:55.339915] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:55.339930] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:55.339937] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:55.341322] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=28][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:55.347600] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.347621] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:55.347632] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.347642] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.347655] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.358837] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF6D-0-0] [lt=6] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1614) [2024-08-16 10:19:55.358886] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF6D-0-0] [lt=30] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=7, avg_log_submit_to_flush_time=647, avg_log_submit_to_slide_time=91014) [2024-08-16 10:19:55.358905] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF6D-0-0] [lt=17][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509803, log_task={header:{begin_lsn:{lsn:18544214006}, end_lsn:{lsn:18544214128}, log_id:53509803, min_scn:{val:1723774794872386003, v:0}, max_scn:{val:1723774794872386003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544213893}, prev_proposal_id:7, committed_end_lsn:{lsn:18544214006}, data_checksum:1913811101, accum_checksum:1256635573, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774795258786, freeze_ts:1723774795258786, submit_ts:1723774795258792, flushed_ts:1723774795259430, gen_to_freeze cost time:0, gen_to_submit cost time:6, submit_to_flush cost time:638}, fs_cb_begin_ts=1723774795358873, log_life_time=100087) [2024-08-16 10:19:55.358920] INFO [PALF] read (log_cache.cpp:100) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=36] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1614, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:55.383492] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=20] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:19:55.396044] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.396067] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:19:55.396096] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=26] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.396107] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.396122] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.399895] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [2562][T1002_TabletShe][T1002][Y0-0000000000000000-0-0] [lt=11] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:19:55.399928] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:99) [2562][T1002_TabletShe][T1002][Y0-0000000000000000-0-0] [lt=21] [emptytablet] succeed to change tablet to empty shell(ret=0, ret="OB_SUCCESS", ls_cnt=4, times=12265) [2024-08-16 10:19:55.415445] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=27] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1, need_refresh=false, gts_local_cache={srr:[mts=1723774794914830], gts:1723774794915014000, latest_srr:[mts=1723774795415413]}) [2024-08-16 10:19:55.415484] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=27] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_refresh=false, gts_local_cache={srr:[mts=1723774795218951], gts:1723774795217660000, latest_srr:[mts=1723774795415474]}) [2024-08-16 10:19:55.415738] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=8] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_refresh=false, gts_local_cache={srr:[mts=1723774795035048], gts:1723774795033247000, latest_srr:[mts=1723774795415727]}) [2024-08-16 10:19:55.424622] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=20] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:19:55.424727] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=20] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:19:55.424958] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=22] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=5) [2024-08-16 10:19:55.439878] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:55.439978] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:55.440008] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:55.440016] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:55.441404] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=34][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:55.447695] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.447718] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:19:55.447727] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.447738] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.447766] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.448483] INFO [STORAGE.TRANS] generate_new_version (ob_tenant_weak_read_server_version_mgr.cpp:120) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=33] [WRS] update tenant weak read server version(tenant_id=1001, server_version={version:{val:1723774790913914000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0, epoch_tstamp:1723774795448468}, version_delta=4534568) [2024-08-16 10:19:55.466145] INFO [STORAGE] runTimerTask (ob_locality_manager.cpp:708) [31516][LocalityReload][T0][Y0-0000000000000000-0-0] [lt=26] runTimer to refresh locality_info(ret=0) [2024-08-16 10:19:55.468290] INFO [SHARE] set_server_locality_array (ob_server_locality_cache.cpp:346) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=18] set_server_locality_array success(server_locality_array=[{inited:true, addr:"192.168.0.124:2882", zone:"zone1", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1713946685756789, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.125:2882", zone:"zone1", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1720514289430943, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.126:2882", zone:"zone1", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1722932391226462, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.127:2882", zone:"zone2", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1713947174711570, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.128:2882", zone:"zone2", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1720514727760575, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.129:2882", zone:"zone2", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1722932391566566, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.131:2882", zone:"zone3", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1714007235497482, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.133:2882", zone:"zone3", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1720516415939965, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.134:2882", zone:"zone3", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1722932391301527, server_stop_time:0, server_status:1}], has_readonly_zone=false) [2024-08-16 10:19:55.468364] INFO [SHARE] load_region (ob_locality_table_operator.cpp:346) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=57] get_primary_zone_prioriry(primary_zone=zone1,zone2,zone3, tenant_primary_region_array=[{region:"sys_region", region_priority:0, zone_array:["zone1", "zone2", "zone3"]}], locality_info={version:0, local_region:"sys_region", local_zone:"zone2", local_idc:"default_idc", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone1", "zone2", "zone3"]}], locality_zone_array:[]}) [2024-08-16 10:19:55.468396] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=31] add_locality_zone success(locality_zone={tenant_id:1, region_priority:0}) [2024-08-16 10:19:55.468414] INFO [SHARE] load_region (ob_locality_table_operator.cpp:346) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=8] get_primary_zone_prioriry(primary_zone=zone1,zone2,zone3, tenant_primary_region_array=[{region:"sys_region", region_priority:0, zone_array:["zone1", "zone2", "zone3"]}], locality_info={version:0, local_region:"sys_region", local_zone:"zone2", local_idc:"default_idc", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone1", "zone2", "zone3"]}], locality_zone_array:[{tenant_id:1, region_priority:0}]}) [2024-08-16 10:19:55.468431] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=16] add_locality_zone success(locality_zone={tenant_id:1001, region_priority:0}) [2024-08-16 10:19:55.468445] INFO [SHARE] load_region (ob_locality_table_operator.cpp:346) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=6] get_primary_zone_prioriry(primary_zone=zone1,zone2,zone3, tenant_primary_region_array=[{region:"sys_region", region_priority:0, zone_array:["zone1", "zone2", "zone3"]}], locality_info={version:0, local_region:"sys_region", local_zone:"zone2", local_idc:"default_idc", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone1", "zone2", "zone3"]}], locality_zone_array:[{tenant_id:1, region_priority:0}, {tenant_id:1001, region_priority:0}]}) [2024-08-16 10:19:55.468478] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=32] add_locality_zone success(locality_zone={tenant_id:1002, region_priority:0}) [2024-08-16 10:19:55.468491] INFO [SHARE] load_region (ob_locality_table_operator.cpp:373) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=6] load region(ret=0, locality_info={version:0, local_region:"sys_region", local_zone:"zone2", local_idc:"default_idc", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone1", "zone2", "zone3"]}], locality_zone_array:[{tenant_id:1, region_priority:0}, {tenant_id:1001, region_priority:0}, {tenant_id:1002, region_priority:0}]}) [2024-08-16 10:19:55.469330] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [31651][T1_PalfGC][T1][Y0-0000000000000000-0-0] [lt=27] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}) [2024-08-16 10:19:55.475455] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [2767][T1001_LockWaitM][T1001][Y0-0000000000000000-0-0] [lt=26] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-08-16 10:19:55.483364] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:55.483644] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=46] Wash time detail, (compute_wash_size_time=136, refresh_score_time=72, wash_time=16) [2024-08-16 10:19:55.485562] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7DB-0-0] [lt=30] [RS_MGR] force set rs list(master_rs="192.168.0.126:2882") [2024-08-16 10:19:55.485591] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7DB-0-0] [lt=22] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"192.168.0.128:2882", server_id:16, rs_addr:"192.168.0.126:2882", rs_server_status:1, epoch_id:11}, hb_response={zone:"zone2", server:"192.168.0.128:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1720514727760575, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=30) [2024-08-16 10:19:55.486843] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=16] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=125828, clean_num=62914) [2024-08-16 10:19:55.495959] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=19] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1002, server_version={version:{val:1723774794872386003, v:0}, total_part_count:4, valid_inner_part_count:0, valid_user_part_count:4}, server_version_delta=623568, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.124:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29408305, succ_cluster_heartbeat_count=29408295, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774715049399002, v:0}, local_cluster_delta=80446555, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:19:55.496095] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.496115] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:55.496124] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.496133] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.496147] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.524728] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=34] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=6840, mem_used=12478464) [2024-08-16 10:19:55.532226] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=58] start do ls ha handler(ls_id_array_=[{id:1079}, {id:1261}, {id:1262}, {id:1260}]) [2024-08-16 10:19:55.536647] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 444.00) [2024-08-16 10:19:55.536709] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=56] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:55.536771] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=28] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.99, rt: 454.75) [2024-08-16 10:19:55.536783] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: MICRO_BLOCK_CACHE_IO, mode: read, size: 6436.57, iops: 13.95, rt: 268.21) [2024-08-16 10:19:55.536790] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:55.536809] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=14] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536820] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536826] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536833] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536839] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536845] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536851] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536857] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536863] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536869] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536876] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536882] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536888] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536894] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536900] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.536906] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:55.539965] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:55.540101] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=30] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:55.540118] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:55.540125] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:55.541507] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:55.547764] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.547784] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:55.547795] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.547804] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.547817] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.549053] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A55-0-0] [lt=44] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:19:55.549599] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=527, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:19:55.549630] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=31, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:19:55.549692] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=37, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:19:55.559483] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1211) [2024-08-16 10:19:55.584035] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=10] ====== check clog disk timer task ====== [2024-08-16 10:19:55.584067] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=29] get_disk_usage(ret=0, capacity(MB):=2150, used(MB):=0) [2024-08-16 10:19:55.584104] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=30] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=676457349, need_update_checkpoint_scn=false) [2024-08-16 10:19:55.585219] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=6] ====== check clog disk timer task ====== [2024-08-16 10:19:55.585240] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=19] get_disk_usage(ret=0, capacity(MB):=0, used(MB):=0) [2024-08-16 10:19:55.585253] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=8] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=0, need_update_checkpoint_scn=false) [2024-08-16 10:19:55.590710] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=9][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:19:55.592916] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=24] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:19:55.593008] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=23][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:19:55.593030] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=21][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:19:55.593039] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=8] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:19:55.596097] INFO [STORAGE.TRANS] generate_weak_read_timestamp_ (ob_ls_wrs_handler.cpp:188) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=51] get wrs ts(ls_id={id:1079}, delta=222200, log_service_ts=1723774795373291, min_tx_service_ts=4611686018427387, timestamp={val:1723774795373291000, v:0}) [2024-08-16 10:19:55.596125] INFO [STORAGE.TRANS] print_stat_info (ob_keep_alive_ls_handler.cpp:214) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=23] [Keep Alive Stat] LS Keep Alive Info(tenant_id=1002, LS_ID={id:1079}, Not_Master_Cnt=100, Near_To_GTS_Cnt=0, Other_Error_Cnt=0, Submit_Succ_Cnt=0, last_scn="{val:1723774794872386002, v:0}", last_lsn={lsn:18544213949}, last_gts={val:0, v:0}, min_start_scn="{val:0, v:0}", min_start_status=1, sys_ls_end_scn={val:0, v:0}) [2024-08-16 10:19:55.596141] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.596153] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] handle task succ(id={id:1260}) [2024-08-16 10:19:55.596189] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=33] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.596206] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.596221] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.606513] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=22] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:19:55.607084] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [25973][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=28] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:19:55.616395] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=24] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60644, mem_used=137263104) [2024-08-16 10:19:55.625066] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=21] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=4) [2024-08-16 10:19:55.640039] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:55.640177] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:55.640200] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:55.640208] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:55.641359] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE54B-0-0] [lt=49] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=5, avg_log_gen_to_submit_time=60, avg_log_submit_to_flush_time=677, avg_log_submit_to_slide_time=1229) [2024-08-16 10:19:55.641580] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=26][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:55.645361] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=13] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:19:55.645394] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=24] [tabletchange] task check ls(ls->get_ls_id()={id:1079}, tablet_persist_trigger=0) [2024-08-16 10:19:55.645413] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=16] [tabletchange] task check ls(ls->get_ls_id()={id:1260}, tablet_persist_trigger=0) [2024-08-16 10:19:55.645420] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=6] [tabletchange] task check ls(ls->get_ls_id()={id:1261}, tablet_persist_trigger=0) [2024-08-16 10:19:55.645424] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [tabletchange] task check ls(ls->get_ls_id()={id:1262}, tablet_persist_trigger=0) [2024-08-16 10:19:55.645428] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:228) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [tabletchange] succeed to gc_tablet(ret=0, ret="OB_SUCCESS", ls_cnt=4, times=11132) [2024-08-16 10:19:55.647833] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.647854] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:55.647864] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.647873] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.647887] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.650152] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7430B-0-0] [lt=22] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:19:55.650185] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7430B-0-0] [lt=32] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1022, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:19:55.659065] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF7F-0-0] [lt=8][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509806, log_task={header:{begin_lsn:{lsn:18544214372}, end_lsn:{lsn:18544214494}, log_id:53509806, min_scn:{val:1723774795373291001, v:0}, max_scn:{val:1723774795373291001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544214250}, prev_proposal_id:7, committed_end_lsn:{lsn:18544214372}, data_checksum:1913811101, accum_checksum:1628657642, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774795558843, freeze_ts:1723774795558843, submit_ts:1723774795558847, flushed_ts:1723774795559535, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:688}, fs_cb_begin_ts=1723774795659052, log_life_time=100209) [2024-08-16 10:19:55.687434] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:55.687751] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=33] Wash time detail, (compute_wash_size_time=181, refresh_score_time=85, wash_time=16) [2024-08-16 10:19:55.691138] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=188742, clean_num=62914) [2024-08-16 10:19:55.696221] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.696246] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:19:55.696263] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.696277] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.696304] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.702181] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [31775][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=10] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:19:55.702209] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:101) [31775][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=21] [emptytablet] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12636) [2024-08-16 10:19:55.740136] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:55.740275] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=30] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=45, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:19:55.740327] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:19:55.740349] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:55.740364] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:55.740373] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:55.741565] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [2674][T1001_LogLoop][T1001][Y0-0000000000000000-0-0] [lt=54] LogLoopThread round_cost_time(us)(round_cost_time=3) [2024-08-16 10:19:55.741657] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:55.747943] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.747964] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:55.747978] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.748047] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.748089] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.759099] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF85-0-0] [lt=70] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:19:55.759150] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF85-0-0] [lt=22] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=9, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:19:55.759162] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF85-0-0] [lt=11] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=91033, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:19:55.759344] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=33] ====== tenant freeze timer task ====== [2024-08-16 10:19:55.759377] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=20][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:19:55.773603] INFO [PALF] read (log_cache.cpp:100) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1667, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:55.796297] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.796274] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=20] get_max_applied_scn(scn={val:1723774795413914002, v:0}, this={ls_id_:{id:1260}, role_:1, proposal_id_:29, palf_committed_end_lsn_:{lsn:8828132908}, last_check_scn_:{val:1723774795413914003, v:0}, max_applied_cb_scn_:{val:1723774795413914002, v:0}}) [2024-08-16 10:19:55.796318] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:55.796329] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.796339] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.796353] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.796319] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=40] get_min_unreplayed_log_info(lsn={lsn:8469424174}, scn={val:1723617929041268003, v:0}, this={ls_id_:{id:1260}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723617929662670, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:2710542, ret_code_:0}}, next_to_submit_lsn_:{lsn:8469424174}, next_to_submit_scn_:{val:1723617929041268003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516550107534003, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69fc05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8469424052}, end_lsn:{lsn:8469424174}, read_buf:{buf_len_:3682304, buf_:0x7fa69fc05000}, block_size:67104768, log_storage_:0x7fa6bd55b470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723617929041268002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723617929041268002, v:0}, data_checksum:3690972251, flag:0}}, init_mode_version:0, accumulate_checksum:2828504552, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723617929041268002, v:0}}}}}) [2024-08-16 10:19:55.796373] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=46] get_max_applied_scn(scn={val:1723617908966009002, v:0}, this={ls_id_:{id:1261}, role_:2, proposal_id_:4, palf_committed_end_lsn_:{lsn:32276646160}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1723617908966009002, v:0}}) [2024-08-16 10:19:55.796392] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=15] get_min_unreplayed_log_info(lsn={lsn:32601556918}, scn={val:1723774795356230005, v:0}, this={ls_id_:{id:1261}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774795773570, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:344950, ret_code_:0}}, next_to_submit_lsn_:{lsn:32601556918}, next_to_submit_scn_:{val:1723774795356230005, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516552080113001, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd805000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:32601556796}, end_lsn:{lsn:32601556918}, read_buf:{buf_len_:3682304, buf_:0x7fa6cd805000}, block_size:67104768, log_storage_:0x7fa6fe299470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774795356230004, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774795356230004, v:0}, data_checksum:1657047228, flag:1}}, init_mode_version:0, accumulate_checksum:2405917433, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774795356230004, v:0}}}}}) [2024-08-16 10:19:55.796429] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=30] get_max_applied_scn(scn={val:0, v:0}, this={ls_id_:{id:1262}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:0, v:0}}) [2024-08-16 10:19:55.796440] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=8] get_min_unreplayed_log_info(lsn={lsn:8394838486}, scn={val:1723774795249927005, v:0}, this={ls_id_:{id:1262}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774795750192, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10679383, ret_code_:0}}, next_to_submit_lsn_:{lsn:8394838486}, next_to_submit_scn_:{val:1723774795249927005, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516551920929002, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69ec05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8394838364}, end_lsn:{lsn:8394838486}, read_buf:{buf_len_:3682304, buf_:0x7fa69ec05000}, block_size:67104768, log_storage_:0x7fa6be533470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774795249927004, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774795249927004, v:0}, data_checksum:2892311396, flag:0}}, init_mode_version:0, accumulate_checksum:4105949418, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774795249927004, v:0}}}}}) [2024-08-16 10:19:55.825184] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=42] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=3) [2024-08-16 10:19:55.840204] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:55.840330] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:19:55.840359] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:55.840367] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:55.840373] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:55.840961] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:19:55.841743] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:55.847995] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.848015] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:55.848025] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.848035] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.848055] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.848712] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=24] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1001, server_version={version:{val:1723774790913914000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0}, server_version_delta=4934795, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.124:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29431289, succ_cluster_heartbeat_count=29431277, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774715706314000, v:0}, local_cluster_delta=80142395, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:19:55.853906] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2752][T1001_TenantMet][T1001][Y0-0000000000000000-0-0] [lt=90] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=0, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:19:55.857619] INFO [SERVER] try_reload_schema (ob_server_schema_updater.cpp:440) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=12] schedule fetch new schema task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:19:55.857653] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:255) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E7B-0-0] [lt=8] [REFRESH_SCHEMA] start to process schema refresh task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:19:55.857662] INFO [SERVER] do_heartbeat_event (ob_heartbeat.cpp:268) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=41] try reload schema success(schema_version=1722933080671200, refresh_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_ret=0) [2024-08-16 10:19:55.857675] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:280) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E7B-0-0] [lt=18] [REFRESH_SCHEMA] local schema info is newer, no need to refresh schema(ret=0, ret="OB_SUCCESS", local_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:19:55.859122] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF89-0-0] [lt=5][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509809, log_task={header:{begin_lsn:{lsn:18544215019}, end_lsn:{lsn:18544215141}, log_id:53509809, min_scn:{val:1723774795569369002, v:0}, max_scn:{val:1723774795569369002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544214616}, prev_proposal_id:7, committed_end_lsn:{lsn:18544215019}, data_checksum:1913811101, accum_checksum:2902298629, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774795759093, freeze_ts:1723774795759093, submit_ts:1723774795759123, flushed_ts:1723774795759807, gen_to_freeze cost time:0, gen_to_submit cost time:30, submit_to_flush cost time:684}, fs_cb_begin_ts=1723774795859111, log_life_time=100018) [2024-08-16 10:19:55.859735] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=595, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:19:55.869956] INFO [SERVER] reload_tenant_task (ob_tenant_tablet_ttl_mgr.cpp:1297) [20386][T1002_TTLTablet][T1002][YB42C0A80080-00061CCC99C6C549-0-0] [lt=37] finish reload tenant task(local_tenant_task_={tenant_id:1002, task_id:-1, is_usr_trigger:false, need_check:false, is_dirty:false, ttl_continue:true, rsp_time:-1, state:4, is_reused:true}, tenant_task={gmt_create:0, gmt_modified:0, tenant_id:18446744073709551615, table_id:18446744073709551615, tablet_id:18446744073709551615, task_id:0, task_start_time:0, task_update_time:0, trigger_type:0, status:19, ttl_del_cnt:0, max_version_del_cnt:0, scan_cnt:0, row_key:"", ret_code:"OB_SUCCESS"}, is_leader=true) [2024-08-16 10:19:55.873679] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74314-0-0] [lt=35] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=609, avg_log_submit_to_slide_time=999) [2024-08-16 10:19:55.880848] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1044) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=281] success to update min reserved snapshot(reserved_snapshot=1723772990629965000, duration=1800, snapshot_gc_ts_=1723774790629965000) [2024-08-16 10:19:55.882036] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1071) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=18] update reserved snapshot finished(cost_ts=1191, reserved_snapshot=1723772990629965000) [2024-08-16 10:19:55.891588] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=24] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:55.891840] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=124, refresh_score_time=66, wash_time=16) [2024-08-16 10:19:55.895032] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=16] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=251656, clean_num=62914) [2024-08-16 10:19:55.896383] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.896405] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:55.896415] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.896425] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.896439] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.901374] INFO [SHARE] get_server_tenant_configs (ob_unit_getter.cpp:231) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=15] get_server_tenant_configs(unit_infos=[{unit:unit_id:1014, resource_pool_id:1006, unit_group_id:1004, zone:"zone2", server:"192.168.0.128:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:8, max_cpu:8, memory_size:"7GB", log_disk_size:"21GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, pool:{resource_pool_id:1006, name:"pool_epean_zone2_qcd", unit_count:2, unit_config_id:1059, zone_list:["zone2"], tenant_id:1002, replica_type:0}}]) [2024-08-16 10:19:55.901419] INFO [SHARE] get_compat_mode (ob_unit_getter.cpp:810) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=42] jx_debug: get tenant compatibility mode(tenant_id=1002, compat_mode=0) [2024-08-16 10:19:55.901435] INFO [SHARE] divide_meta_tenant (ob_unit_resource.cpp:658) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=10] divide meta tenant resource finish(ret=0, ret="OB_SUCCESS", meta_resource={min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"2.1GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}, user_resource={min_cpu:8, max_cpu:8, memory_size:"6GB", log_disk_size:"18.9GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}, unit_resource={min_cpu:8, max_cpu:8, memory_size:"7GB", log_disk_size:"21GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}) [2024-08-16 10:19:55.901455] INFO [SHARE] divide_meta_tenant (ob_unit_getter.cpp:125) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=18] divide meta tenant finish(ret=0, ret="OB_SUCCESS", meta_tenant_config={tenant_id:1001, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"2.1GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:0, is_removed:false}, user_config={tenant_id:1002, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:8, max_cpu:8, memory_size:"6GB", log_disk_size:"18.9GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:0, is_removed:false}) [2024-08-16 10:19:55.901565] INFO [COMMON] update_io_clock (ob_io_mclock.cpp:454) [31598][OmtNodeBalancer][T1002][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=27] update other group clock success(index=0, unit_config={min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, cur_config={deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}) [2024-08-16 10:19:55.901590] INFO [COMMON] update_basic_io_config (ob_io_manager.cpp:918) [31598][OmtNodeBalancer][T1002][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=21] update basic io config success(tenant_id_=1002, io_config_={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:0, max_percent:0, weight_percent:0}]}, io_clock_={is_inited_:true, group_clocks:[], other_clock:{is_inited_:true, is_stopped_:false, reservation_clock:{iops:9223372036854775807, last_ns:1723774795228088000}, is_unlimited:false, limitation_clock:{iops:9223372036854775807, last_ns:1723774795228088000}, proportion_clock:{iops:8, last_ns:8227596373099699712}}, unit_clock:{iops:9223372036854775807, last_ns:1723774795228088000}, io_config_:{group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, io_usage_:{doing_request_count:[0:0]}}) [2024-08-16 10:19:55.901706] INFO [COMMON] update_io_clock (ob_io_mclock.cpp:454) [31598][OmtNodeBalancer][T1001][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=37] update other group clock success(index=0, unit_config={min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, cur_config={deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}) [2024-08-16 10:19:55.901733] INFO [COMMON] update_basic_io_config (ob_io_manager.cpp:918) [31598][OmtNodeBalancer][T1001][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=26] update basic io config success(tenant_id_=1001, io_config_={group_num_:0, memory_limit_:1073741824, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, io_config={group_num_:0, memory_limit_:1073741824, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:0, max_percent:0, weight_percent:0}]}, io_clock_={is_inited_:true, group_clocks:[], other_clock:{is_inited_:true, is_stopped_:false, reservation_clock:{iops:9223372036854775807, last_ns:0}, is_unlimited:false, limitation_clock:{iops:9223372036854775807, last_ns:0}, proportion_clock:{iops:8, last_ns:0}}, unit_clock:{iops:9223372036854775807, last_ns:0}, io_config_:{group_num_:0, memory_limit_:1073741824, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, io_usage_:{doing_request_count:[0:0]}}) [2024-08-16 10:19:55.901774] INFO [SERVER.OMT] refresh_hidden_sys_memory (ob_tenant_node_balancer.cpp:427) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=31] update hidden sys tenant memory succeed (allowed_mem_limit=1073741824) [2024-08-16 10:19:55.905899] INFO [SHARE] refresh_resource_plan (ob_resource_plan_manager.cpp:122) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=13] refresh resource plan success(tenant_id=1, plan_name=, directives=[]) [2024-08-16 10:19:55.908657] INFO [SHARE] refresh_resource_user_mapping_rule (ob_resource_mapping_rule_manager.cpp:159) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=31] refresh resource user mapping rule(tenant_id=1, plan=, user_rules=[]) [2024-08-16 10:19:55.910737] INFO [SHARE] refresh_resource_function_mapping_rule (ob_resource_mapping_rule_manager.cpp:196) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=19] refresh_resource_function_mapping_rule(tenant_id=1, plan=, rules=[]) [2024-08-16 10:19:55.910767] INFO [SHARE] refresh_resource_mapping_rule (ob_resource_mapping_rule_manager.cpp:100) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=28] refresh resource mapping rule success(tenant_id=1, plan=) [2024-08-16 10:19:55.912801] INFO [SQL.DTL] calc_max_buffer (ob_dtl_fc_server.cpp:159) [31598][OmtNodeBalancer][T1][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=9] trace tenant dfc parameters(max_parallel_cnt_=20, max_blocked_buffer_size_=6553600, max_buffer_size_=13107200) [2024-08-16 10:19:55.912866] INFO [SQL.DTL] auto_free_on_time (ob_dtl_tenant_mem_manager.cpp:232) [31598][OmtNodeBalancer][T1][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=22] auto free to reserve buffer count(reserve_buffer_min_size=16777216, max_reserve_count=2, buffer_size=65536, ret=0) [2024-08-16 10:19:55.912898] INFO [SQL.DTL] clean_on_timeout (ob_dtl_fc_server.cpp:131) [31598][OmtNodeBalancer][T1][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=29] tenant dfc status(ret=0, get_tenant_id()=1, get_channel_cnt()=0, get_current_buffer_used()=0, get_current_blocked_cnt()=0, get_current_buffer_cnt()=0, get_max_parallel()=20, get_max_blocked_buffer_size()=6553600, get_max_buffer_size()=13107200, get_accumulated_blocked_cnt()=11, get_max_size_per_channel()=655360) [2024-08-16 10:19:55.916197] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=47] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:19:55.919237] INFO [SHARE] refresh_resource_plan (ob_resource_plan_manager.cpp:122) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=30] refresh resource plan success(tenant_id=1001, plan_name=, directives=[]) [2024-08-16 10:19:55.923461] INFO [SHARE] refresh_resource_user_mapping_rule (ob_resource_mapping_rule_manager.cpp:159) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=20] refresh resource user mapping rule(tenant_id=1001, plan=, user_rules=[]) [2024-08-16 10:19:55.926036] INFO [SHARE] refresh_resource_function_mapping_rule (ob_resource_mapping_rule_manager.cpp:196) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=26] refresh_resource_function_mapping_rule(tenant_id=1001, plan=, rules=[]) [2024-08-16 10:19:55.926092] INFO [SHARE] refresh_resource_mapping_rule (ob_resource_mapping_rule_manager.cpp:100) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=54] refresh resource mapping rule success(tenant_id=1001, plan=) [2024-08-16 10:19:55.930242] INFO [SQL.DTL] calc_max_buffer (ob_dtl_fc_server.cpp:159) [31598][OmtNodeBalancer][T1001][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=12] trace tenant dfc parameters(max_parallel_cnt_=10, max_blocked_buffer_size_=3276800, max_buffer_size_=6553600) [2024-08-16 10:19:55.930301] INFO [SQL.DTL] auto_free_on_time (ob_dtl_tenant_mem_manager.cpp:232) [31598][OmtNodeBalancer][T1001][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=24] auto free to reserve buffer count(reserve_buffer_min_size=16777216, max_reserve_count=2, buffer_size=65536, ret=0) [2024-08-16 10:19:55.930319] INFO [SQL.DTL] clean_on_timeout (ob_dtl_fc_server.cpp:131) [31598][OmtNodeBalancer][T1001][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=16] tenant dfc status(ret=0, get_tenant_id()=1001, get_channel_cnt()=0, get_current_buffer_used()=0, get_current_blocked_cnt()=0, get_current_buffer_cnt()=0, get_max_parallel()=10, get_max_blocked_buffer_size()=3276800, get_max_buffer_size()=6553600, get_accumulated_blocked_cnt()=0, get_max_size_per_channel()=655360) [2024-08-16 10:19:55.934474] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=8] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1002) [2024-08-16 10:19:55.934499] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=21] start lib cache evict(tenant_id=1002, mem_hold=279150592, mem_limit=322122545, cache_obj_num=644, cache_node_num=575) [2024-08-16 10:19:55.934511] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=10] end lib cache evict(tenant_id=1002, cache_evict_num=0, mem_hold=279150592, mem_limit=322122545, cache_obj_num=644, cache_node_num=575) [2024-08-16 10:19:55.934519] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=7] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:19:55.934574] INFO [SHARE] refresh_resource_plan (ob_resource_plan_manager.cpp:122) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=11] refresh resource plan success(tenant_id=1002, plan_name=, directives=[]) [2024-08-16 10:19:55.937211] INFO [SHARE] refresh_resource_user_mapping_rule (ob_resource_mapping_rule_manager.cpp:159) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=27] refresh resource user mapping rule(tenant_id=1002, plan=, user_rules=[]) [2024-08-16 10:19:55.938034] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=9] Dumping All Cache Objs(alloc_obj_list.count()=644, alloc_obj_list=[{obj_id:197058, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179184}, {obj_id:246944, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:166013}, {obj_id:246946, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175596}, {obj_id:199478, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205804}, {obj_id:249042, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:250289, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174834}, {obj_id:250293, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187797}, {obj_id:250294, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158389}, {obj_id:252064, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179845}, {obj_id:252089, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252117, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188403}, {obj_id:252210, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252318, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191338}, {obj_id:252323, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182962}, {obj_id:252333, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252334, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252336, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252340, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252341, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:252345, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221400}, {obj_id:252346, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284685}, {obj_id:252349, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181199}, {obj_id:252359, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:356776}, {obj_id:252372, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180356}, {obj_id:252377, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252386, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252404, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:382127}, {obj_id:252405, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252406, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:252409, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230382}, {obj_id:252417, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277038}, {obj_id:252425, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158366}, {obj_id:252428, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252440, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213028}, {obj_id:252445, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178214}, {obj_id:252447, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:252458, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:941831}, {obj_id:252459, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252464, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157801}, {obj_id:252475, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176302}, {obj_id:252485, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181511}, {obj_id:252488, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203010}, {obj_id:252489, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197827}, {obj_id:252495, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181747}, {obj_id:252502, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:705639}, {obj_id:252503, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:355214}, {obj_id:252511, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294158}, {obj_id:252513, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217004}, {obj_id:252540, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256131}, {obj_id:252541, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294112}, {obj_id:252554, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219188}, {obj_id:252569, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203562}, {obj_id:252573, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174535}, {obj_id:252586, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179957}, {obj_id:252587, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:254547}, {obj_id:252603, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294123}, {obj_id:252660, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196529}, {obj_id:252664, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294182}, {obj_id:252670, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292094}, {obj_id:252671, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294048}, {obj_id:252673, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211150}, {obj_id:252699, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202759}, {obj_id:252702, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252720, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157744}, {obj_id:252721, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:273791}, {obj_id:252730, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189375}, {obj_id:252749, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:399910}, {obj_id:252750, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:314035}, {obj_id:252764, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299083}, {obj_id:252765, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179021}, {obj_id:252769, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:355879}, {obj_id:252771, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188178}, {obj_id:252786, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237378}, {obj_id:252841, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177408}, {obj_id:252849, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252858, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180522}, {obj_id:252859, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180179}, {obj_id:252862, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6708112}, {obj_id:252864, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252865, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232017}, {obj_id:252868, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232266}, {obj_id:252869, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220840}, {obj_id:252939, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252955, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175994}, {obj_id:252962, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252964, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:349147}, {obj_id:252975, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252980, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252994, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252995, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252998, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253001, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253005, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253006, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253007, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253008, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294264}, {obj_id:253009, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292151}, {obj_id:253015, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298926}, {obj_id:253022, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301178}, {obj_id:253025, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298998}, {obj_id:253035, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298892}, {obj_id:253039, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:351399}, {obj_id:253040, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292087}, {obj_id:253047, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298992}, {obj_id:253050, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:288707}, {obj_id:253053, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204401}, {obj_id:253059, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208911}, {obj_id:253060, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211382}, {obj_id:253061, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200421}, {obj_id:253065, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223579}, {obj_id:253071, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290322}, {obj_id:253072, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250919}, {obj_id:253087, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253090, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:323458}, {obj_id:253091, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272373}, {obj_id:253094, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200393}, {obj_id:253095, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187302}, {obj_id:253096, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206572}, {obj_id:253097, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190787}, {obj_id:253098, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177031}, {obj_id:253102, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272582}, {obj_id:253107, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272526}, {obj_id:253113, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213908}, {obj_id:253114, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201513}, {obj_id:253115, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185332}, {obj_id:253122, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240210}, {obj_id:253124, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245608}, {obj_id:253127, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:372694}, {obj_id:253137, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1957477}, {obj_id:253169, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:270222}, {obj_id:253170, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272447}, {obj_id:253174, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:795070}, {obj_id:253234, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:283358}, {obj_id:253282, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253312, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286729}, {obj_id:253341, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253444, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157703}, {obj_id:253450, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186097}, {obj_id:253451, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:287939}, {obj_id:253452, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258575}, {obj_id:253454, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277108}, {obj_id:253457, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218998}, {obj_id:253458, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:285701}, {obj_id:253459, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253468, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181399}, {obj_id:253471, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:874259}, {obj_id:253473, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232858}, {obj_id:253476, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253477, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253478, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207070}, {obj_id:253480, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250296}, {obj_id:253485, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253486, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201045}, {obj_id:253487, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201415}, {obj_id:253488, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202980}, {obj_id:253489, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225769}, {obj_id:253490, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201728}, {obj_id:253491, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197763}, {obj_id:253497, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253500, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253506, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:11947420}, {obj_id:253507, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228298}, {obj_id:253508, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176153}, {obj_id:253509, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223501}, {obj_id:253510, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176542}, {obj_id:253511, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:253512, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218568}, {obj_id:253513, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177328}, {obj_id:253514, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177907}, {obj_id:253515, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184844}, {obj_id:253516, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253517, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177168}, {obj_id:253518, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:253519, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253520, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:253521, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:253522, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:253523, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253524, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253525, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253526, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253527, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:162334}, {obj_id:253529, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253530, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181547}, {obj_id:253531, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181340}, {obj_id:253532, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192983}, {obj_id:253533, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253534, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186898}, {obj_id:253535, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180995}, {obj_id:253536, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253537, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253538, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:253539, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253540, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223911}, {obj_id:253541, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253542, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221984}, {obj_id:253543, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186900}, {obj_id:253544, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213278}, {obj_id:253545, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6705734}, {obj_id:253546, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6739530}, {obj_id:253547, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202652}, {obj_id:253548, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253549, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:236673}, {obj_id:253550, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253551, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253553, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253554, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:367141}, {obj_id:253555, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189381}, {obj_id:253556, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253557, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253558, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205876}, {obj_id:253559, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253560, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189377}, {obj_id:253561, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253562, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237070}, {obj_id:253564, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205271}, {obj_id:253565, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238959}, {obj_id:253566, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292100}, {obj_id:253567, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:287824}, {obj_id:253568, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6741367}, {obj_id:253570, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187799}, {obj_id:253571, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253572, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175560}, {obj_id:253573, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175558}, {obj_id:253574, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:160948}, {obj_id:253576, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253578, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253579, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253580, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235030}, {obj_id:253581, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234962}, {obj_id:253582, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200857}, {obj_id:253583, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253584, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175142}, {obj_id:253585, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174672}, {obj_id:253588, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292043}, {obj_id:253589, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253590, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253591, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253594, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292025}, {obj_id:253595, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253596, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253601, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:383372}, {obj_id:253605, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292063}, {obj_id:253608, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290453}, {obj_id:253610, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253611, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:369179}, {obj_id:253614, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282531}, {obj_id:253615, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292282}, {obj_id:253616, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253617, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197701}, {obj_id:253618, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:476477}, {obj_id:253621, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297717}, {obj_id:253628, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:244896}, {obj_id:253630, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203464}, {obj_id:253631, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196891}, {obj_id:253632, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:253633, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186088}, {obj_id:253634, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181148}, {obj_id:253635, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253638, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:268654}, {obj_id:253640, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201594}, {obj_id:253641, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:265278}, {obj_id:253642, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290670}, {obj_id:253645, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253649, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292134}, {obj_id:253650, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206516}, {obj_id:253653, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290821}, {obj_id:253655, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238325}, {obj_id:253658, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198080}, {obj_id:253659, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216604}, {obj_id:253660, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178528}, {obj_id:253661, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:243178}, {obj_id:253664, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291977}, {obj_id:253665, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216113}, {obj_id:253666, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242569}, {obj_id:253667, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253668, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207760}, {obj_id:253669, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253672, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292096}, {obj_id:253677, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178509}, {obj_id:253681, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292081}, {obj_id:253682, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182898}, {obj_id:253683, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225569}, {obj_id:253691, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208341}, {obj_id:253692, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211962}, {obj_id:253694, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:379579}, {obj_id:253695, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185463}, {obj_id:253696, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253697, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253698, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194454}, {obj_id:253699, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185281}, {obj_id:253700, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277153}, {obj_id:253701, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196647}, {obj_id:253702, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220331}, {obj_id:253703, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253704, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228523}, {obj_id:253705, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184067}, {obj_id:253706, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175893}, {obj_id:253707, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189895}, {obj_id:253708, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251032}, {obj_id:253709, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189423}, {obj_id:253710, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:334817}, {obj_id:253711, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189681}, {obj_id:253712, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209440}, {obj_id:253713, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188174}, {obj_id:253714, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253715, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253716, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174832}, {obj_id:253723, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:932120}, {obj_id:253724, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:262469}, {obj_id:253725, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:332574}, {obj_id:253726, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240754}, {obj_id:253727, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210719}, {obj_id:253728, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253729, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:253730, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185730}, {obj_id:253731, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250192}, {obj_id:253732, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177398}, {obj_id:253733, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191786}, {obj_id:253734, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177488}, {obj_id:253735, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276369}, {obj_id:253736, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219016}, {obj_id:253737, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:510255}, {obj_id:253740, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197323}, {obj_id:253743, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234930}, {obj_id:253747, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210144}, {obj_id:253749, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253750, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253751, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227821}, {obj_id:253753, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:302184}, {obj_id:253754, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253755, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183280}, {obj_id:253756, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253757, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282524}, {obj_id:253758, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200791}, {obj_id:253759, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:384669}, {obj_id:253760, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:382379}, {obj_id:253761, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:873919}, {obj_id:253762, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:870651}, {obj_id:253763, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:516499}, {obj_id:253764, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253765, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253766, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204396}, {obj_id:253767, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222427}, {obj_id:253768, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253769, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221512}, {obj_id:253770, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:554188}, {obj_id:253771, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247245}, {obj_id:253772, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253773, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196589}, {obj_id:253774, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187792}, {obj_id:253775, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198057}, {obj_id:253776, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279996}, {obj_id:253777, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221914}, {obj_id:253779, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180817}, {obj_id:253780, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179884}, {obj_id:253782, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225027}, {obj_id:253784, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250315}, {obj_id:253791, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292016}, {obj_id:253792, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291532}, {obj_id:253793, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180590}, {obj_id:253796, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250098}, {obj_id:253797, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180228}, {obj_id:253798, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176856}, {obj_id:253799, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195067}, {obj_id:253800, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253801, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177751}, {obj_id:253802, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208961}, {obj_id:253805, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240180}, {obj_id:253808, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256449}, {obj_id:253809, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:449159}, {obj_id:253810, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188388}, {obj_id:253811, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199543}, {obj_id:253812, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195231}, {obj_id:253813, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253814, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253815, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214158}, {obj_id:253816, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253817, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:288727}, {obj_id:253820, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291994}, {obj_id:253821, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235062}, {obj_id:253827, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175823}, {obj_id:253828, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174483}, {obj_id:253829, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217784}, {obj_id:253830, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177814}, {obj_id:253831, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:444256}, {obj_id:253832, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195622}, {obj_id:253833, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253834, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:497014}, {obj_id:253835, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253836, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:261672}, {obj_id:253837, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:435156}, {obj_id:253838, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:516719}, {obj_id:253839, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200167}, {obj_id:253840, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271961}, {obj_id:253841, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194596}, {obj_id:253842, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205257}, {obj_id:253843, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174397}, {obj_id:253845, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253846, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253847, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253848, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224798}, {obj_id:253849, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223872}, {obj_id:253850, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224553}, {obj_id:253851, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221103}, {obj_id:253852, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219500}, {obj_id:253853, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219425}, {obj_id:253854, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253855, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240470}, {obj_id:253856, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253857, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253858, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220267}, {obj_id:253859, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220212}, {obj_id:253861, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290476}, {obj_id:253864, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:547504}, {obj_id:253865, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233724}, {obj_id:253866, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218340}, {obj_id:253867, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197559}, {obj_id:253868, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253869, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217547}, {obj_id:253870, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235957}, {obj_id:253871, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237233}, {obj_id:253872, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:280008}, {obj_id:253873, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208674}, {obj_id:253874, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190144}, {obj_id:253875, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240637}, {obj_id:253876, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187604}, {obj_id:253878, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241654}, {obj_id:253879, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253880, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253881, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176921}, {obj_id:253882, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188072}, {obj_id:253883, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:246455}, {obj_id:253884, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176040}, {obj_id:253886, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253887, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253890, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, adde [2024-08-16 10:19:55.938948] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=889] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:19:55.939330] INFO [SHARE] refresh_resource_function_mapping_rule (ob_resource_mapping_rule_manager.cpp:196) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=24] refresh_resource_function_mapping_rule(tenant_id=1002, plan=, rules=[]) [2024-08-16 10:19:55.939358] INFO [SHARE] refresh_resource_mapping_rule (ob_resource_mapping_rule_manager.cpp:100) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=26] refresh resource mapping rule success(tenant_id=1002, plan=) [2024-08-16 10:19:55.939941] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [31787][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=18] tx gc loop thread is running(MTL_ID()=1) [2024-08-16 10:19:55.940282] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:55.940385] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1332) [2024-08-16 10:19:55.940408] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:55.940422] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:55.940429] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:55.941460] INFO [SQL.DTL] calc_max_buffer (ob_dtl_fc_server.cpp:159) [31598][OmtNodeBalancer][T1002][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=10] trace tenant dfc parameters(max_parallel_cnt_=80, max_blocked_buffer_size_=26214400, max_buffer_size_=52428800) [2024-08-16 10:19:55.941523] INFO [SQL.DTL] auto_free_on_time (ob_dtl_tenant_mem_manager.cpp:232) [31598][OmtNodeBalancer][T1002][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=25] auto free to reserve buffer count(reserve_buffer_min_size=16777216, max_reserve_count=2, buffer_size=65536, ret=0) [2024-08-16 10:19:55.941542] INFO [SQL.DTL] clean_on_timeout (ob_dtl_fc_server.cpp:131) [31598][OmtNodeBalancer][T1002][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=17] tenant dfc status(ret=0, get_tenant_id()=1002, get_channel_cnt()=0, get_current_buffer_used()=0, get_current_blocked_cnt()=0, get_current_buffer_cnt()=0, get_max_parallel()=80, get_max_blocked_buffer_size()=26214400, get_max_buffer_size()=52428800, get_accumulated_blocked_cnt()=21568, get_max_size_per_channel()=655360) [2024-08-16 10:19:55.941567] INFO [SERVER.OMT] run1 (ob_tenant_node_balancer.cpp:109) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=9] refresh tenant units(sys_unit_cnt=3, units=[{tenant_id:1002, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:8, max_cpu:8, memory_size:"6GB", log_disk_size:"18.9GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}, {tenant_id:1001, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"2.1GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}], ret=0, ret="OB_SUCCESS") [2024-08-16 10:19:55.941607] INFO set_nway (ob_tenant_mutil_allocator.cpp:335) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=36] finish set nway(tenant_id_=1002, nway=8) [2024-08-16 10:19:55.941622] INFO update_tenant_mem_limit (ob_tenant_mutil_allocator_mgr.cpp:380) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=14] ObTenantMutilAllocator update tenant mem_limit finished(ret=0, tenant_id=1002, nway=8, new_tma_limit=3221225450, pre_tma_limit=3221225450, cur_memstore_limit_percent=50, tenant_config={tenant_id:1002, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:8, max_cpu:8, memory_size:"6GB", log_disk_size:"18.9GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}) [2024-08-16 10:19:55.941643] INFO update_tenant_mem_limit (ob_tenant_mutil_allocator_mgr.cpp:391) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=17] succ to set_memstore_threshold of memstore allocator(tenant_id=1002, ret=0) [2024-08-16 10:19:55.941649] INFO set_nway (ob_tenant_mutil_allocator.cpp:335) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=4] finish set nway(tenant_id_=1001, nway=1) [2024-08-16 10:19:55.941653] INFO update_tenant_mem_limit (ob_tenant_mutil_allocator_mgr.cpp:380) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=4] ObTenantMutilAllocator update tenant mem_limit finished(ret=0, tenant_id=1001, nway=1, new_tma_limit=536870900, pre_tma_limit=536870900, cur_memstore_limit_percent=50, tenant_config={tenant_id:1001, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"2.1GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}) [2024-08-16 10:19:55.941664] INFO update_tenant_mem_limit (ob_tenant_mutil_allocator_mgr.cpp:391) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=9] succ to set_memstore_threshold of memstore allocator(tenant_id=1001, ret=0) [2024-08-16 10:19:55.941827] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:55.943264] INFO [SERVER] cal_all_part_disk_default_percentage (ob_server_utils.cpp:325) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=3] cal_all_part_disk_default_percentage succ(data_dir="/home/admin/oceanbase/store/ep_test_cluster/sstable", clog_dir="/home/admin/oceanbase/store/ep_test_cluster/clog", shared_mode=true, data_disk_total_size=258645426176, data_disk_default_percentage=60, clog_disk_total_size=258645426176, clog_disk_default_percentage=30) [2024-08-16 10:19:55.943289] INFO [SERVER] decide_disk_size (ob_server_utils.cpp:361) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=23] decide disk size finished(suggested_disk_size=0, suggested_disk_percentage=0, default_disk_percentage=30, total_space=258645426176, disk_size=77593627852) [2024-08-16 10:19:55.943299] INFO [SERVER] get_log_disk_info_in_config (ob_server_utils.cpp:88) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=9] get_log_disk_info_in_config(suggested_data_disk_size=0, suggested_clog_disk_size=0, suggested_data_disk_percentage=0, suggested_clog_disk_percentage=0, log_disk_size=77593627852, log_disk_percentage=0, total_log_disk_size=258645426176) [2024-08-16 10:19:55.943308] INFO [CLOG] try_resize (ob_server_log_block_mgr.cpp:785) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=6] try_resize success(ret=0, log_disk_size=77593627852, total_log_disk_size=258645426176, this={dir::"/home/admin/oceanbase/store/ep_test_cluster/clog/log_pool", dir_fd:14, meta_fd:15, log_pool_meta:{curr_total_size:77577846784, next_total_size:77577846784, status:0}, min_block_id:1108, max_block_id:2020, min_log_disk_size_for_all_tenants_:22548578304, is_inited:true}) [2024-08-16 10:19:55.943320] INFO [SERVER.OMT] run1 (ob_tenant_node_balancer.cpp:133) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26E-0-0] [lt=11] refresh tenant config(tenants=[1, 1001, 1002], ret=0) [2024-08-16 10:19:55.948064] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=44] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.948100] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=34] handle task succ(id={id:1260}) [2024-08-16 10:19:55.948108] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.948116] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.948132] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:55.949556] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=34] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:55.949582] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=19] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:55.949813] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:19:55.949858] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=38][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:19:55.949867] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:19:55.949874] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:19:55.950286] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:19:55.950846] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74315-0-0] [lt=19] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=10, avg_log_submit_to_flush_time=609, avg_log_submit_to_slide_time=1053) [2024-08-16 10:19:55.950887] INFO [PALF] read (log_cache.cpp:100) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:55.961213] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=18] ====== check clog disk timer task ====== [2024-08-16 10:19:55.961245] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=23] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:55.961273] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=11] cannot_recycle_log_size statistics(cannot_recycle_log_size=391614191, threshold=6088116142, need_update_checkpoint_scn=false) [2024-08-16 10:19:55.970772] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007C-000616D3CE4B9708-0-0] [lt=46][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:55.982820] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [31597][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=529] dump tenant info(tenant={id:1, tenant_meta:{unit:{tenant_id:1, unit_id:1000, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1000, name:"hidden_sys_unit", resource:{min_cpu:2, max_cpu:2, memory_size:"1GB", log_disk_size:"0GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2}}, mode:0, create_timestamp:1722932515520653, is_removed:false}, super_block:{tenant_id:1, replay_start_point:ObLogCursor{file_id=2, log_id=5654, offset=657873}, ls_meta_entry:[-1](ver=0,mode=0,seq=0), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:true, version:2}, create_status:1}, unit_min_cpu:"2.000000000000000000e+00", unit_max_cpu:"2.000000000000000000e+00", total_worker_cnt:38, min_worker_cnt:22, max_worker_cnt:150, stopped:0, worker_us:236153719841, recv_hp_rpc_cnt:2505780, recv_np_rpc_cnt:1588818, recv_lp_rpc_cnt:0, recv_mysql_cnt:14351616, recv_task_cnt:796856, recv_large_req_cnt:0, tt_large_quries:1456, pop_normal_cnt:6817887325, workers:22, nesting workers:8, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=456 cnt[3]=0 cnt[4]=0 cnt[5]=61103922 cnt[6]=13 cnt[7]=2 cnt[8]=0 cnt[9]=0 , group_map:group_id = 2,queue_size = 0,recv_req_cnt = 1829,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 2,token_change = 1722932514913013 group_id = 10,queue_size = 0,recv_req_cnt = 26884293,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 2,token_change = 1722932514916132 group_id = 15,queue_size = 0,recv_req_cnt = 5403,min_worker_cnt = 2,max_worker_cnt = 2,worker_cnt = 0,token_change = 1723771320518099 group_id = 5,queue_size = 0,recv_req_cnt = 64,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 2,token_change = 1722932514916435 group_id = 19,queue_size = 0,recv_req_cnt = 5,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 0,token_change = 1722932502279864 group_id = 100,queue_size = 0,recv_req_cnt = 275,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 0,token_change = 1723726142832261 group_id = 9,queue_size = 0,recv_req_cnt = 14309,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 2,token_change = 1722932514916640 group_id = 17,queue_size = 0,recv_req_cnt = 1151,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 0,token_change = 1722931691277014 group_id = 1,queue_size = 0,recv_req_cnt = 59213,min_worker_cnt = 8,max_worker_cnt = 8,worker_cnt = 0,token_change = 1722932561115577 , rpc_stat_info: pcode=0x4a9:cnt=90 pcode=0x51f:cnt=31 pcode=0x152:cnt=5, token_change_ts:1722932514916861, tenant_role:1}) [2024-08-16 10:19:55.983364] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [31597][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=543] dump tenant info(tenant={id:508, tenant_meta:{unit:{tenant_id:508, unit_id:1000, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1000, name:"virtual_tenant_unit", resource:{min_cpu:5, max_cpu:5, memory_size:"1GB", log_disk_size:"2GB", min_iops:10000, max_iops:50000, iops_weight:0}}, mode:0, create_timestamp:1720514721373193, is_removed:false}, super_block:{tenant_id:508, replay_start_point:ObLogCursor{file_id=1, log_id=1, offset=0}, ls_meta_entry:[-1](ver=0,mode=0,seq=0), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:true, version:2}, create_status:1}, unit_min_cpu:"5.000000000000000000e+00", unit_max_cpu:"5.000000000000000000e+00", total_worker_cnt:52, min_worker_cnt:52, max_worker_cnt:150, stopped:0, worker_us:47267687885, recv_hp_rpc_cnt:0, recv_np_rpc_cnt:20599003, recv_lp_rpc_cnt:0, recv_mysql_cnt:0, recv_task_cnt:0, recv_large_req_cnt:0, tt_large_quries:0, pop_normal_cnt:16195581303, workers:52, nesting workers:0, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=0 cnt[3]=0 cnt[4]=0 cnt[5]=0 cnt[6]=0 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 15,queue_size = 0,recv_req_cnt = 26,min_worker_cnt = 2,max_worker_cnt = 2,worker_cnt = 0,token_change = 1723040524903572 , rpc_stat_info: pcode=0x5c0:cnt=53, token_change_ts:1720514721381040, tenant_role:0}) [2024-08-16 10:19:55.983863] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [31597][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=498] dump tenant info(tenant={id:1001, tenant_meta:{unit:{tenant_id:1001, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"2.1GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}, super_block:{tenant_id:1001, replay_start_point:ObLogCursor{file_id=1, log_id=1, offset=0}, ls_meta_entry:[-1](ver=0,mode=0,seq=0), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:false, version:2}, create_status:1}, unit_min_cpu:"1.000000000000000000e+00", unit_max_cpu:"1.000000000000000000e+00", total_worker_cnt:7, min_worker_cnt:6, max_worker_cnt:150, stopped:0, worker_us:7614837679, recv_hp_rpc_cnt:14124963, recv_np_rpc_cnt:3166720, recv_lp_rpc_cnt:0, recv_mysql_cnt:0, recv_task_cnt:0, recv_large_req_cnt:0, tt_large_quries:4775, pop_normal_cnt:1317876300, workers:6, nesting workers:0, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=0 cnt[3]=0 cnt[4]=0 cnt[5]=0 cnt[6]=0 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 5,queue_size = 0,recv_req_cnt = 78,min_worker_cnt = 1,max_worker_cnt = 150,worker_cnt = 1,token_change = 1720909340702942 , rpc_stat_info: pcode=0x743:cnt=19 pcode=0x651:cnt=10 pcode=0x745:cnt=9, token_change_ts:1720516534273175, tenant_role:1}) [2024-08-16 10:19:55.984358] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [31597][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=494] dump tenant info(tenant={id:1002, tenant_meta:{unit:{tenant_id:1002, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:8, max_cpu:8, memory_size:"6GB", log_disk_size:"18.9GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}, super_block:{tenant_id:1002, replay_start_point:ObLogCursor{file_id=65, log_id=18553178, offset=36846227}, ls_meta_entry:[44114](ver=0,mode=0,seq=226903), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:false, version:2}, create_status:1}, unit_min_cpu:"8.000000000000000000e+00", unit_max_cpu:"8.000000000000000000e+00", total_worker_cnt:60, min_worker_cnt:34, max_worker_cnt:150, stopped:0, worker_us:432347668518, recv_hp_rpc_cnt:4509851, recv_np_rpc_cnt:5675644, recv_lp_rpc_cnt:0, recv_mysql_cnt:16147287, recv_task_cnt:159913, recv_large_req_cnt:0, tt_large_quries:27992571, pop_normal_cnt:8315032560, workers:34, nesting workers:8, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=2588102 cnt[3]=0 cnt[4]=0 cnt[5]=5584 cnt[6]=1591 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 2,queue_size = 0,recv_req_cnt = 58190529,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 8,token_change = 1722932655869139 group_id = 15,queue_size = 0,recv_req_cnt = 5401,min_worker_cnt = 2,max_worker_cnt = 2,worker_cnt = 0,token_change = 1723771320558792 group_id = 5,queue_size = 0,recv_req_cnt = 67,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 8,token_change = 1722932655869453 group_id = 19,queue_size = 0,recv_req_cnt = 698939,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 1,token_change = 1723774783862238 group_id = 100,queue_size = 0,recv_req_cnt = 11824,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 0,token_change = 1723774359377709 group_id = 17,queue_size = 0,recv_req_cnt = 8013,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 0,token_change = 1723452012952496 group_id = 16,queue_size = 0,recv_req_cnt = 425,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 0,token_change = 1723646582548182 group_id = 1,queue_size = 0,recv_req_cnt = 327998946,min_worker_cnt = 8,max_worker_cnt = 8,worker_cnt = 1,token_change = 1723774780652594 , rpc_stat_info: pcode=0x1501:cnt=306 pcode=0x1502:cnt=204 pcode=0x1516:cnt=201 pcode=0x150a:cnt=60 pcode=0x150b:cnt=60, token_change_ts:1722932655869739, tenant_role:1}) [2024-08-16 10:19:55.996481] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:55.996505] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:19:55.996516] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:55.996527] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:55.996553] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.011895] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=15] gc stale ls task succ [2024-08-16 10:19:56.011941] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=10] gc stale ls task succ [2024-08-16 10:19:56.011968] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=17] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:19:56.011992] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=16] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:19:56.012005] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=10] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:19:56.016633] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:56.016805] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:56.025302] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=26] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=2) [2024-08-16 10:19:56.026446] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=19] PNIO [ratelimit] time: 1723774796026444, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007596, add_bytes: 0 [2024-08-16 10:19:56.040352] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:56.040513] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:56.040530] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:56.040540] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:56.041160] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1332) [2024-08-16 10:19:56.041904] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:56.048135] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.048155] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:56.048165] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.048174] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.048188] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.048923] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=54] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:19:56.053223] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80080-00061CCCC6B436BF-0-0] [lt=23][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:56.054550] INFO [STORAGE.TRANS] print_stat_ (ob_black_list.cpp:398) [31842][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=102] start to print blacklist info [2024-08-16 10:19:56.054730] INFO [STORAGE.TRANS] run1 (ob_black_list.cpp:194) [31842][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=17] ls blacklist refresh finish(cost_time=9811) [2024-08-16 10:19:56.059396] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF8F-0-0] [lt=19][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509810, log_task={header:{begin_lsn:{lsn:18544215141}, end_lsn:{lsn:18544215263}, log_id:53509810, min_scn:{val:1723774795812789000, v:0}, max_scn:{val:1723774795812789000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544215019}, prev_proposal_id:7, committed_end_lsn:{lsn:18544215141}, data_checksum:1913811101, accum_checksum:2997402467, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774795859094, freeze_ts:1723774795859095, submit_ts:1723774795859098, flushed_ts:1723774795859781, gen_to_freeze cost time:1, gen_to_submit cost time:4, submit_to_flush cost time:683}, fs_cb_begin_ts=1723774796059381, log_life_time=200287) [2024-08-16 10:19:56.059499] INFO [LIB] stat (utility.h:1197) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=45] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=29, stat_interval=1000000, avg (count/cost)=38, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:19:56.059531] INFO [LIB] stat (utility.h:1197) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=31] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=29, stat_interval=1000000, avg (count/cost)=14, this=0x7fa713668b28, extra_info="") [2024-08-16 10:19:56.060108] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=37, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:19:56.064565] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:19:56.064588] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:19:56.072778] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=31] PNIO [ratelimit] time: 1723774796072777, bytes: 1496435778412, bw: 0.173343 MB/s, add_ts: 1000822, add_bytes: 181913 [2024-08-16 10:19:56.072822] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8598-0-0] [lt=39] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:19:56.074013] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE559-0-0] [lt=20] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1667) [2024-08-16 10:19:56.084511] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AA9-0-0] [lt=19] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:19:56.084554] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AA9-0-0] [lt=25] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774795940560001, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774795940560001, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:19:56.095467] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:56.095720] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash time detail, (compute_wash_size_time=121, refresh_score_time=66, wash_time=16) [2024-08-16 10:19:56.096473] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=59][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:19:56.096550] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.096584] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=30] handle task succ(id={id:1260}) [2024-08-16 10:19:56.096698] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=29] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.100684] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=3980] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.101210] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=513] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.103870] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=314570, clean_num=62914) [2024-08-16 10:19:56.115193] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7D6-0-0] [lt=116] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:19:56.140451] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:56.141502] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=35] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:56.141537] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=32] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:56.141545] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:56.141976] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:56.142291] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=607, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:19:56.142329] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=33, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:19:56.142374] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:19:56.146405] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [2553][T1002_ObXAHbWor][T1002][Y0-0000000000000000-0-0] [lt=57] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-08-16 10:19:56.148205] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.148229] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] handle task succ(id={id:1260}) [2024-08-16 10:19:56.148237] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.148246] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.148261] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.149509] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A61-0-0] [lt=19] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:19:56.149531] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A61-0-0] [lt=20] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:19:56.150025] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=6746948, this={dio_aligned_buf:{buf_write_offset:958, buf_padding_size:3138, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:44277508, truncate_used_ts:10955916}, log_block_size:67108864, dir_fd:203, io_fd:100}, aligned_buf_len=4096, aligned_buf="", aligned_block_offset=6746112, buf_len=122, total_write_size_=13519, total_write_size_after_dio_=430080, ob_pwrite_used_ts=58209, count=101, each_pwrite_cost=576) [2024-08-16 10:19:56.156553] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=18] ====== tenant freeze timer task ====== [2024-08-16 10:19:56.163277] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAD3-0-0] [lt=16][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=72, len=311, pos=281, bitmap=NULL) [2024-08-16 10:19:56.172928] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD859A-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1667, avg_group_log_size=166) [2024-08-16 10:19:56.183136] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=46] gc stale ls task succ [2024-08-16 10:19:56.196636] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=36] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.196663] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:19:56.196673] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.196684] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.196698] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.199092] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE7B-0-0] [lt=17] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:19:56.199140] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE7B-0-0] [lt=33] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774795951284001, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774795951284001, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:19:56.223296] INFO [PALF] runTimerTask (block_gc_timer_task.cpp:101) [2433][T1002_PalfGC][T1002][Y0-0000000000000000-0-0] [lt=47] BlockGCTimerTask success(ret=0, cost_time_us=17, palf_env_impl_={IPalfEnvImpl:{IPalfEnvImpl:"Dummy"}, self:"192.168.0.128:2882", log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002", disk_options_wrapper:{disk_opts_for_stopping_writing:{log_disk_size(MB):19353, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):19353, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):629, sequence:4}, log_alloc_mgr_:{flying_log_task:0, flying_meta_task:0}}) [2024-08-16 10:19:56.225410] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=26] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=1) [2024-08-16 10:19:56.230282] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=19] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574608, offset=42139425}, end_cursor:ObLogCursor{file_id=65, log_id=18574609, offset=42139666}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751638050, len:4319, log_data_len:241, seq:18574608, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:56.230863] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=41] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574609, offset=42139666}, end_cursor:ObLogCursor{file_id=65, log_id=18574610, offset=42139915}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7519b4050, len:4078, log_data_len:249, seq:18574609, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:56.232330] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BC9E-0-0] [lt=29] push gts task success(*task={:""}) [2024-08-16 10:19:56.236150] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=16] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:56.236294] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:56.236439] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:56.236476] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=23] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:56.236966] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:56.237168] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=18] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:56.237248] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:56.239607] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BC9E-0-0] [lt=18] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:19:56.240514] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:56.240613] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:56.240628] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:56.240635] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:56.241126] INFO [STORAGE.DUP_TABLE] print_tablet_diag_info_log (ob_dup_table_tablets.cpp:2125) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=7] [DUP_TABLE_DIAG: Dup Tablet Info] tenant: 1002, ls: 1079, is_master: False, total_tablet_count: 100, need_confirm_new_set_count: 0, readable_set_count: 9, tablet_set_print_buf_used: 2423/131072, tablet_id_print_buf_used:4604/1048576, last_readable_log_entry_scn: {val:1723774782981623002, v:0}, last_readable_sync_succ_time_: 1723774783249818, [DUP_TABLE_DIAG: Readable Dup Tablet Set] unique_id = 10, tablet_count = 1, change_status = {flag_:5, tablet_change_scn_:{val:1719901770359004000, v:0}, need_confirm_scn_:{val:1719901770359004000, v:0}, readable_version_:{val:1719901770359004000, v:0}, trx_ref_:0} [DUP_TABLE_DIAG: Readable Dup Tablet Set] unique_id = 11, tablet_count = 6, change_status = {flag_:5, tablet_change_scn_:{val:1719901806391470000, v:0}, need_confirm_scn_:{val:1719901806391470000, v:0}, readable_version_:{val:1719901806391470000, v:0}, trx_ref_:0} [DUP_TABLE_DIAG: Readable Dup Tablet Set] unique_id = 14, tablet_count = 55, change_status = {flag_:5, tablet_change_scn_:{val:1720489809538874001, v:0}, need_confirm_scn_:{val:1720489809538874001, v:0}, readable_version_:{val:1720489809538874001, v:0}, trx_ref_:0} [DUP_TABLE_DIAG: Readable Dup Tablet Set] unique_id = 15, tablet_count = 6, change_status = {flag_:5, tablet_change_scn_:{val:1720489821551676003, v:0}, need_confirm_scn_:{val:1720489821551676003, v:0}, readable_version_:{val:1720489821551676003, v:0}, trx_ref_:0} [DUP_TABLE_DIAG: Readable Dup Tablet Set] unique_id = 21, tablet_count = 4, change_status = {flag_:5, tablet_change_scn_:{val:1720600305166516002, v:0}, need_confirm_scn_:{val:1720600305166516002, v:0}, readable_version_:{val:1720600305166516002, v:0}, trx_ref_:0} [DUP_TABLE_DIAG: Readable Dup Tablet Set] unique_id = 26, tablet_count = 2, change_status = {flag_:5, tablet_change_scn_:{val:1721960037133277000, v:0}, need_confirm_scn_:{val:1721960037133277000, v:0}, readable_version_:{val:1721960037133277000, v:0}, trx_ref_:0} [DUP_TABLE_DIAG: Readable Dup Tablet Set] unique_id = 27, tablet_count = 4, change_status = {flag_:5, tablet_change_scn_:{val:1721997285551914000, v:0}, need_confirm_scn_:{val:1721997285551914000, v:0}, readable_version_:{val:1721997285551914000, v:0}, trx_ref_:0} [DUP_TABLE_DIAG: Readable Dup Tablet Set] unique_id = 32, tablet_count = 20, change_status = {flag_:5, tablet_change_scn_:{val:1722425155146440643, v:0}, need_confirm_scn_:{val:1722425155146440643, v:0}, readable_version_:{val:1722425155146440643, v:0}, trx_ref_:0} [DUP_TABLE_DIAG: Readable Dup Tablet Set] unique_id = 35, tablet_count = 2, change_status = {flag_:5, tablet_change_scn_:{val:1722510651516707000, v:0}, need_confirm_scn_:{val:1722510651516707000, v:0}, readable_version_:{val:1722510651516707000, v:0}, trx_ref_:0} [DUP_TABLE_DIAG: Tablet Set Member - from 10] { TabletID = 212125 } [DUP_TABLE_DIAG: Tablet Set Member - from 11] { TabletID = 1152921504606895583 } { TabletID = 1152921504606895581 } { TabletID = 1152921504606895582 } { TabletID = 1152921504606895579 } { TabletID = 1152921504606895580 } [DUP_TABLE_DIAG: Tablet Set Member - from 11] { TabletID = 1152921504606895578 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 1152921504606884618 } { TabletID = 208204 } { TabletID = 1152921504606884602 } { TabletID = 1152921504606884579 } { TabletID = 1152921504606884608 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 208203 } { TabletID = 1152921504606884580 } { TabletID = 1152921504606884595 } { TabletID = 1152921504606884610 } { TabletID = 1152921504606884577 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 1152921504606884578 } { TabletID = 1152921504606884591 } { TabletID = 1152921504606884604 } { TabletID = 1152921504606884582 } { TabletID = 208217 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 1152921504606884585 } { TabletID = 1152921504606884589 } { TabletID = 1152921504606884605 } { TabletID = 1152921504606884590 } { TabletID = 1152921504606884588 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 1152921504606884599 } { TabletID = 1152921504606884594 } { TabletID = 1152921504606884583 } { TabletID = 1152921504606884617 } { TabletID = 1152921504606884615 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 208216 } { TabletID = 1152921504606884609 } { TabletID = 1152921504606884586 } { TabletID = 1152921504606884614 } { TabletID = 1152921504606884616 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 208209 } { TabletID = 1152921504606884606 } { TabletID = 208212 } { TabletID = 208207 } { TabletID = 1152921504606884597 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 208208 } { TabletID = 208205 } { TabletID = 1152921504606884598 } { TabletID = 1152921504606884596 } { TabletID = 208210 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 1152921504606884584 } { TabletID = 1152921504606884619 } { TabletID = 1152921504606884613 } { TabletID = 208215 } { TabletID = 208211 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 1152921504606884592 } { TabletID = 1152921504606884581 } { TabletID = 1152921504606884601 } { TabletID = 1152921504606884587 } { TabletID = 208206 } [DUP_TABLE_DIAG: Tablet Set Member - from 14] { TabletID = 1152921504606884603 } { TabletID = 1152921504606884600 } { TabletID = 208214 } { TabletID = 1152921504606884607 } { TabletID = 1152921504606884593 } [DUP_TABLE_DIAG: Tablet Set Member - from 15] { TabletID = 1152921504606884624 } { TabletID = 1152921504606884623 } { TabletID = 1152921504606884621 } { TabletID = 208218 } { TabletID = 1152921504606884620 } [DUP_TABLE_DIAG: Tablet Set Member - from 15] { TabletID = 1152921504606884622 } [DUP_TABLE_DIAG: Tablet Set Member - from 21] { TabletID = 214096 } { TabletID = 1152921504606903842 } { TabletID = 1152921504606903840 } { TabletID = 1152921504606903841 } [DUP_TABLE_DIAG: Tablet Set Member - from 26] { TabletID = 1152921504606913971 } { TabletID = 219872 } [DUP_TABLE_DIAG: Tablet Set Member - from 27] { TabletID = 1152921504606914007 } { TabletID = 1152921504606914006 } { TabletID = 1152921504606914005 } { TabletID = 219884 } [DUP_TABLE_DIAG: Tablet Set Member - from 32] { TabletID = 1152921504606884625 } { TabletID = 208290 } { TabletID = 208219 } { TabletID = 1152921504606884840 } { TabletID = 1152921504606884826 } [DUP_TABLE_DIAG: Tablet Set Member - from 32] { TabletID = 1152921504606884839 } { TabletID = 1152921504606884827 } { TabletID = 1152921504606884836 } { TabletID = 1152921504606884627 } { TabletID = 1152921504606884825 } [DUP_TABLE_DIAG: Tablet Set Member - from 32] { TabletID = 1152921504606884838 } { TabletID = 1152921504606884828 } { TabletID = 1152921504606884830 } { TabletID = 1152921504606884829 } { TabletID = 208288 } [DUP_TABLE_DIAG: Tablet Set Member - from 32] { TabletID = 208220 } { TabletID = 1152921504606884824 } { TabletID = 1152921504606884831 } { TabletID = 1152921504606884832 } { TabletID = 1152921504606884837 } [DUP_TABLE_DIAG: Tablet Set Member - from 35] { TabletID = 1152921504606914395 } { TabletID = 220116 } [2024-08-16 10:19:56.241755] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7352A-0-0] [lt=39] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:19:56.241774] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7352A-0-0] [lt=18] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=1358, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:19:56.241984] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=21] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574610, offset=42139915}, end_cursor:ObLogCursor{file_id=65, log_id=18574611, offset=42140164}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751690050, len:3829, log_data_len:249, seq:18574610, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:56.242092] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=29][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:56.242621] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=23] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574611, offset=42140164}, end_cursor:ObLogCursor{file_id=65, log_id=18574612, offset=42140413}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75166c050, len:3580, log_data_len:249, seq:18574611, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:56.242975] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [31653][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=18] LogLoopThread round_cost_time(us)(round_cost_time=2) [2024-08-16 10:19:56.248293] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.248313] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:56.248321] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.248330] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.248342] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.259438] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF9B-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1501, avg_group_log_size=150) [2024-08-16 10:19:56.259478] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEF9B-0-0] [lt=24][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509812, log_task={header:{begin_lsn:{lsn:18544215385}, end_lsn:{lsn:18544215507}, log_id:53509812, min_scn:{val:1723774795873944001, v:0}, max_scn:{val:1723774795873944001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544215263}, prev_proposal_id:7, committed_end_lsn:{lsn:18544215385}, data_checksum:1913811101, accum_checksum:2831152583, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774796159405, freeze_ts:1723774796159405, submit_ts:1723774796159409, flushed_ts:1723774796160154, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:745}, fs_cb_begin_ts=1723774796259467, log_life_time=100062) [2024-08-16 10:19:56.273533] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=560, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:19:56.273558] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1098) [2024-08-16 10:19:56.273578] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=28, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:19:56.273621] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=40, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:19:56.274038] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE55F-0-0] [lt=32] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:19:56.274052] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE55F-0-0] [lt=13] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=1074, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:19:56.291096] INFO [STORAGE] runTimerTask (ob_tenant_memory_printer.cpp:33) [31393][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=18] === Run print tenant memory usage task === [2024-08-16 10:19:56.291294] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [31393][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=22] [MEMORY] tenant: 500, limit: 9,223,372,036,854,775,807 hold: 944,365,568 rpc_hold: 0 cache_hold: 0 cache_used: 0 cache_item_count: 0 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 316,022,784 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= GLIBC hold_bytes= 27,262,976 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 115,343,360 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= LIBEASY hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= LOGGER_CTX_ID hold_bytes= 12,582,912 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= RPC_CTX_ID hold_bytes= 6,291,456 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PKT_NIO hold_bytes= 12,640,256 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= SCHEMA_SERVICE hold_bytes= 118,247,424 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= UNEXPECTED_IN_500 hold_bytes= 331,780,096 limit= 9,223,372,036,854,775,807 [2024-08-16 10:19:56.291600] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [31393][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=18] [MEMORY] tenant_id= 500 ctx_id= DEFAULT_CTX_ID hold= 316,022,784 used= 285,578,272 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 52,268,992 used= 50,996,736 count= 3,317 avg_used= 15,374 block_cnt= 3,317 chunk_cnt= 31 mod=TZInfoArray [MEMORY] hold= 33,574,912 used= 33,554,464 count= 1 avg_used= 33,554,464 block_cnt= 1 chunk_cnt= 1 mod=BloomFilter [MEMORY] hold= 18,894,848 used= 18,874,392 count= 1 avg_used= 18,874,392 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDTLINT [MEMORY] hold= 14,376,960 used= 12,837,240 count= 195 avg_used= 65,832 block_cnt= 195 chunk_cnt= 26 mod=[T]ObSessionDIB [MEMORY] hold= 10,907,648 used= 10,866,720 count= 13 avg_used= 835,901 block_cnt= 13 chunk_cnt= 9 mod=IoControl [MEMORY] hold= 9,457,664 used= 9,437,784 count= 1 avg_used= 9,437,784 block_cnt= 1 chunk_cnt= 1 mod=HashBuckInteChe [MEMORY] hold= 8,585,216 used= 8,566,048 count= 1 avg_used= 8,566,048 block_cnt= 1 chunk_cnt= 1 mod=MemDumpContext [MEMORY] hold= 6,865,456 used= 6,785,296 count= 48 avg_used= 141,360 block_cnt= 25 chunk_cnt= 11 mod=PartitTableTask [MEMORY] hold= 5,087,232 used= 5,027,008 count= 6 avg_used= 837,834 block_cnt= 6 chunk_cnt= 4 mod=KvstCachWashStr [MEMORY] hold= 4,746,944 used= 4,726,520 count= 2 avg_used= 2,363,260 block_cnt= 2 chunk_cnt= 2 mod=HashBucTenComMo [MEMORY] hold= 4,739,072 used= 4,718,712 count= 1 avg_used= 4,718,712 block_cnt= 1 chunk_cnt= 1 mod=PxP2PDhMgrKey [MEMORY] hold= 4,739,072 used= 4,718,712 count= 1 avg_used= 4,718,712 block_cnt= 1 chunk_cnt= 1 mod=HashPxBlooFilKe [MEMORY] hold= 4,472,832 used= 3,915,776 count= 68 avg_used= 57,584 block_cnt= 68 chunk_cnt= 28 mod=[T]char [MEMORY] hold= 4,307,328 used= 4,278,711 count= 3 avg_used= 1,426,237 block_cnt= 3 chunk_cnt= 3 mod=SqlDtlMgr [MEMORY] hold= 4,231,936 used= 4,203,008 count= 6 avg_used= 700,501 block_cnt= 5 chunk_cnt= 5 mod=BaseLogWriter [MEMORY] hold= 4,214,880 used= 4,194,328 count= 2 avg_used= 2,097,164 block_cnt= 2 chunk_cnt= 2 mod=SerFuncRegHT [MEMORY] hold= 4,046,848 used= 3,969,504 count= 13 avg_used= 305,346 block_cnt= 13 chunk_cnt= 8 mod=DedupQueue [MEMORY] hold= 3,984,128 used= 3,952,128 count= 498 avg_used= 7,936 block_cnt= 498 chunk_cnt= 30 mod=ObTabletLSCache [MEMORY] hold= 3,353,392 used= 3,282,840 count= 640 avg_used= 5,129 block_cnt= 204 chunk_cnt= 40 mod=DeadLock [MEMORY] hold= 2,621,440 used= 2,619,392 count= 16 avg_used= 163,712 block_cnt= 16 chunk_cnt= 9 mod=Iteratorget_tenant_id()=1, queue_size=10000000, config_mem_limit_=67108864, request_manager_->get_size_used()=4964, evict_high_size_level=9000000, evict_low_size_level=8000000, allocator->allocated()=39515136, evict_high_mem_level=46137345, evict_low_mem_level=25165824) [2024-08-16 10:19:56.299053] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=17] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=4964, mem_used=39515136) [2024-08-16 10:19:56.301409] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=19] ====== tenant freeze timer task ====== [2024-08-16 10:19:56.301438] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=18][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:19:56.304365] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:56.304657] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash time detail, (compute_wash_size_time=114, refresh_score_time=87, wash_time=26) [2024-08-16 10:19:56.307865] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=377484, clean_num=62914) [2024-08-16 10:19:56.340601] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:56.340724] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:56.340740] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:56.340748] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:56.342151] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=25][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:56.348362] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.348384] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:56.348394] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.348404] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.348418] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.359777] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFA1-0-0] [lt=6] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1501) [2024-08-16 10:19:56.359815] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFA1-0-0] [lt=21] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=7, avg_log_submit_to_flush_time=677, avg_log_submit_to_slide_time=100118) [2024-08-16 10:19:56.359836] INFO [PALF] read (log_cache.cpp:100) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1501, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:56.359833] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFA1-0-0] [lt=16][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509813, log_task={header:{begin_lsn:{lsn:18544215507}, end_lsn:{lsn:18544215629}, log_id:53509813, min_scn:{val:1723774795873944002, v:0}, max_scn:{val:1723774795873944002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544215385}, prev_proposal_id:7, committed_end_lsn:{lsn:18544215507}, data_checksum:1913811101, accum_checksum:2970825789, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774796259420, freeze_ts:1723774796259420, submit_ts:1723774796259424, flushed_ts:1723774796260051, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:627}, fs_cb_begin_ts=1723774796359802, log_life_time=100382) [2024-08-16 10:19:56.366557] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [2798][T1001_TxLoopWor][T1001][Y0-0000000000000000-0-0] [lt=10] tx gc loop thread is running(MTL_ID()=1001) [2024-08-16 10:19:56.366578] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [2798][T1001_TxLoopWor][T1001][Y0-0000000000000000-0-0] [lt=20] try gc retain ctx [2024-08-16 10:19:56.383581] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=27] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:19:56.396779] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.396802] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:56.396813] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.396823] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.396836] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.405661] INFO pn_ratelimit (group.c:551) [31557][IngressService][T0][Y0-0000000000000000-0-0] [lt=26] PNIO set ratelimit as 9223372036854775807 bytes/s, grp_id=2 [2024-08-16 10:19:56.416970] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=33] gts statistics(tenant_id=1001, gts_rpc_cnt=27, get_gts_cache_cnt=109, get_gts_with_stc_cnt=17, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-08-16 10:19:56.417271] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=26] gts statistics(tenant_id=1002, gts_rpc_cnt=20, get_gts_cache_cnt=390, get_gts_with_stc_cnt=10, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=1, try_wait_gts_elapse_cnt=0) [2024-08-16 10:19:56.417333] INFO [STORAGE.TRANS] alloc (ob_ts_response_handler.cpp:108) [31439][pnio1][T0][YB42C0A80080-00061CCCCD620690-0-0] [lt=21] ts response task statistics(alloc_count=170000266, free_count=85000132) [2024-08-16 10:19:56.417934] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=50][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa69ba14d70, session_id:3225547039, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:false, reader:{fd:234}, err:5, last_decode_time:1723774786671572, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:992, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:19:56.418005] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=58] kill and revert session(conn.sessid_=3225547039, proxy_sessid=13882346384134438915, server_id=16, ret=0) [2024-08-16 10:19:56.418032] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=24] sock ref clean, do destroy(*s={this:0x7fa69ba14d70, session_id:3225547039, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:false, reader:{fd:234}, err:5, last_decode_time:1723774786671572, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:992, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:19:56.418101] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=26] unlock session of tenant(conn.sessid_=3225547039, proxy_sessid=13882346384134438915, conn.tenant_id_=1) [2024-08-16 10:19:56.418140] WDIAG [COMMON] get (ob_link_hashmap.h:313) [31841][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=8][errcode=0] uref leak check(node->uref_=1074538680, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:19:56.418117] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=14] connection close(sessid=3225547039, proxy_sessid=13882346384134438915, tenant_id=1, server_id=16, from_proxy=true, from_java_client=false, c/s protocol="OB_2_0_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCC65F2DFD-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:4, cur_pkt_pos_:10, pkt_rec[0]:{send:obp_mysql_header_{len_:6382, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{send:obp20_header_:{payload_len:14, pkt_seq:0, req_id:13418348, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:496, seq_:2}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[2]:{send:obp20_header_:{payload_len:43, pkt_seq:0, req_id:13418349, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:440, seq_:2}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{receive:obp20_header_:{payload_len:384, pkt_seq:0, req_id:13418350, flag_.flags:58, ext_flags:0}, obp_mysql_header_{len_:380, rec_:384, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[4]:{send:obp_compess_header_{com_len:258, com_seq:1}, obp20_header_:{payload_len:230, pkt_seq:1, req_id:13418350, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_compess_header_{com_len:258, com_seq:1}, obp20_header_:{payload_len:230, pkt_seq:1, req_id:13418350, flag_.flags:10, ext_flags:0}, obp_mysql_header_{pkt_num_:3, seq_:5}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[6]:{send:obp_compess_header_{com_len:258, com_seq:1}, obp20_header_:{payload_len:230, pkt_seq:1, req_id:13418350, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[7]:{send:obp_compess_header_{com_len:258, com_seq:1}, obp20_header_:{payload_len:230, pkt_seq:1, req_id:13418350, flag_.flags:10, ext_flags:0}, obp_mysql_header_{pkt_num_:1, seq_:7}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[8]:{send:obp_compess_header_{com_len:258, com_seq:1}, obp20_header_:{payload_len:230, pkt_seq:1, req_id:13418350, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:9, seq_:8}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[9]:{send:obp_compess_header_{com_len:258, com_seq:1}, obp20_header_:{payload_len:230, pkt_seq:1, req_id:13418350, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:12, seq_:9}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}], disconnect_state=4) [2024-08-16 10:19:56.418219] INFO [SERVER] run (obmp_disconnect.cpp:82) [31841][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=20] free session successfully(sessid=3225547039) [2024-08-16 10:19:56.418242] INFO [SERVER] run (obmp_disconnect.cpp:89) [31841][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=21] mark session id unused(sessid=3225547039) [2024-08-16 10:19:56.424228] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=8] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1) [2024-08-16 10:19:56.424261] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=28] start lib cache evict(tenant_id=1, mem_hold=46137344, mem_limit=53687090, cache_obj_num=123, cache_node_num=123) [2024-08-16 10:19:56.424295] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=31] end lib cache evict(tenant_id=1, cache_evict_num=0, mem_hold=46137344, mem_limit=53687090, cache_obj_num=123, cache_node_num=123) [2024-08-16 10:19:56.424303] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=7] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:19:56.424747] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=44] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:19:56.424849] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=21] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:19:56.425519] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=28] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=0) [2024-08-16 10:19:56.427260] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=4] Dumping All Cache Objs(alloc_obj_list.count()=123, alloc_obj_list=[{obj_id:27, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190650}, {obj_id:95, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:96, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:174, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:984, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205451}, {obj_id:1112, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:310633}, {obj_id:1476, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179375}, {obj_id:1489, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191695}, {obj_id:1687, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198485}, {obj_id:1732, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172160}, {obj_id:1736, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174033}, {obj_id:1741, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175220}, {obj_id:1758, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205820}, {obj_id:1862, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188466}, {obj_id:2017, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:2151, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189565}, {obj_id:2278, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:2279, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189832}, {obj_id:2280, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176478}, {obj_id:2281, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177168}, {obj_id:2282, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174044}, {obj_id:2283, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174081}, {obj_id:2284, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174036}, {obj_id:2285, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185923}, {obj_id:2286, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220611}, {obj_id:2287, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207401}, {obj_id:2288, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180621}, {obj_id:2289, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177481}, {obj_id:2290, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:2291, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183781}, {obj_id:2292, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2293, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188441}, {obj_id:2294, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260630}, {obj_id:2295, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223446}, {obj_id:2296, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173811}, {obj_id:2297, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172864}, {obj_id:2298, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179482}, {obj_id:2299, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186746}, {obj_id:2300, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204032}, {obj_id:2301, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188714}, {obj_id:2302, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183948}, {obj_id:2303, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182890}, {obj_id:2304, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2305, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:2306, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2307, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:2308, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2309, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:2310, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2311, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:2312, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203568}, {obj_id:2313, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191582}, {obj_id:2314, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182638}, {obj_id:2315, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182415}, {obj_id:2316, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184713}, {obj_id:2317, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182209}, {obj_id:2318, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205928}, {obj_id:2319, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223722}, {obj_id:2320, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238225}, {obj_id:2321, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194888}, {obj_id:2322, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220593}, {obj_id:2323, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188416}, {obj_id:2325, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188394}, {obj_id:2327, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196424}, {obj_id:2330, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229007}, {obj_id:2333, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194173}, {obj_id:2336, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210443}, {obj_id:2337, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206506}, {obj_id:2338, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180446}, {obj_id:2339, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234757}, {obj_id:2340, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179743}, {obj_id:2341, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179601}, {obj_id:2342, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184831}, {obj_id:2343, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188563}, {obj_id:2344, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237176}, {obj_id:2345, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198348}, {obj_id:2346, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256258}, {obj_id:2347, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185263}, {obj_id:2348, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196309}, {obj_id:2349, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:265571}, {obj_id:2350, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260809}, {obj_id:2351, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256181}, {obj_id:2352, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299476}, {obj_id:2353, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:270176}, {obj_id:2354, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:305943}, {obj_id:2355, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172528}, {obj_id:2356, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176862}, {obj_id:2357, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183129}, {obj_id:2358, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175603}, {obj_id:2359, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181253}, {obj_id:2360, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181039}, {obj_id:2361, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185755}, {obj_id:2362, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202273}, {obj_id:2363, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:267576}, {obj_id:2364, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183894}, {obj_id:2365, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217508}, {obj_id:2366, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211579}, {obj_id:2367, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184855}, {obj_id:2368, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228602}, {obj_id:2369, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217173}, {obj_id:2370, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214453}, {obj_id:2371, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206378}, {obj_id:2372, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178773}, {obj_id:2373, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177464}, {obj_id:2374, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230030}, {obj_id:2375, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177376}, {obj_id:2376, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183047}, {obj_id:2377, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210108}, {obj_id:2378, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202314}, {obj_id:2379, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202343}, {obj_id:2380, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198396}, {obj_id:2381, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229073}, {obj_id:2382, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204285}, {obj_id:2383, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179141}, {obj_id:2384, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2385, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179385}, {obj_id:2386, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180644}, {obj_id:2387, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2388, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175472}, {obj_id:2389, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175472}, {obj_id:2390, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2391, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174540}, {obj_id:2392, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188792}]) [2024-08-16 10:19:56.427495] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=230] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:19:56.429543] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [31771][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=8] ====== checkpoint timer task ====== [2024-08-16 10:19:56.429566] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [31771][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=16] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:19:56.440674] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:56.440646] INFO [CLOG] get_replay_process (ob_replay_status.cpp:1073) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=17] replay status is not follower(min_unreplayed_lsn={lsn:8469424174}, base_lsn={lsn:0}, this={ls_id_:{id:1260}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:1, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723617929662670, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:2710542, ret_code_:0}}, next_to_submit_lsn_:{lsn:8469424174}, next_to_submit_scn_:{val:1723617929041268003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516550107534003, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69fc05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8469424052}, end_lsn:{lsn:8469424174}, read_buf:{buf_len_:3682304, buf_:0x7fa69fc05000}, block_size:67104768, log_storage_:0x7fa6bd55b470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723617929041268002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723617929041268002, v:0}, data_checksum:3690972251, flag:0}}, init_mode_version:0, accumulate_checksum:2828504552, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723617929041268002, v:0}}}}}) [2024-08-16 10:19:56.440703] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=56] get_replay_process success(id={id:1260}, replayed_log_size=8469424174, unreplayed_log_size=0) [2024-08-16 10:19:56.440714] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=7] get_replay_process success(id={id:1261}, replayed_log_size=32601557528, unreplayed_log_size=0) [2024-08-16 10:19:56.440720] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=3] get_replay_process success(id={id:1262}, replayed_log_size=8394839218, unreplayed_log_size=0) [2024-08-16 10:19:56.440730] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=3] get_replay_process success(id={id:1079}, replayed_log_size=5525890637, unreplayed_log_size=0) [2024-08-16 10:19:56.440747] INFO [CLOG] runTimerTask (ob_log_replay_service.cpp:159) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=3] dump tenant replay process(tenant_id=1002, unreplayed_log_size(MB)=0, estimate_time(second)=0, replayed_log_size(MB)=52444, last_replayed_log_size(MB)=52444, round_cost_time(second)=10, pending_replay_log_size(MB)=0) [2024-08-16 10:19:56.440773] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:56.440795] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:56.440802] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:56.442210] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=29][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:56.448469] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.448500] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=28] handle task succ(id={id:1260}) [2024-08-16 10:19:56.448514] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.448527] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.448543] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.448540] INFO [STORAGE.TRANS] alloc (ob_trans_factory.cpp:141) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E0949-0-0] [lt=0] ObTransCtx statistics(active_coord_ctx_count=0, active_part_ctx_count=1, total_release_part_ctx_count=4) [2024-08-16 10:19:56.448806] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E0949-0-0] [lt=26] current clock is (clock_=0, max_seq_=0, mem_limit=1643328294, cur_mem_hold=289406976, attr_.tenant_id_=1002) [2024-08-16 10:19:56.451705] WDIAG [SERVER] do_process (obmp_query.cpp:1018) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E094B-0-0] [lt=28][errcode=0] need disconnect(ret=0, need_disconnect=true) [2024-08-16 10:19:56.459892] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFA7-0-0] [lt=32][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509814, log_task={header:{begin_lsn:{lsn:18544215629}, end_lsn:{lsn:18544215751}, log_id:53509814, min_scn:{val:1723774795873944003, v:0}, max_scn:{val:1723774795873944003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544215507}, prev_proposal_id:7, committed_end_lsn:{lsn:18544215629}, data_checksum:1913811101, accum_checksum:413553993, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774796359758, freeze_ts:1723774796359758, submit_ts:1723774796359763, flushed_ts:1723774796360491, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:728}, fs_cb_begin_ts=1723774796459879, log_life_time=100121) [2024-08-16 10:19:56.484275] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007D-00061CCCA5024D1B-0-0] [lt=22][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:56.496879] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.496902] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:19:56.496913] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.496923] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.496937] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.505171] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80083-000616E44D0F990B-0-0] [lt=36][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:56.508379] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:56.508656] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash time detail, (compute_wash_size_time=123, refresh_score_time=85, wash_time=17) [2024-08-16 10:19:56.512005] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=440398, clean_num=62914) [2024-08-16 10:19:56.524943] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=39] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=6864, mem_used=12478464) [2024-08-16 10:19:56.532370] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=25] start do ls ha handler(ls_id_array_=[{id:1262}, {id:1079}, {id:1261}, {id:1260}]) [2024-08-16 10:19:56.535830] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [31738][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=68] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:940, allocator used:3737440, allocator total:3859072}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:140, allocator used:9182740, allocator total:9182740}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=13041812, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:63, allocator used:124992, allocator total:130816}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:19:56.540014] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 358.00) [2024-08-16 10:19:56.540046] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=25] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:56.540138] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=55] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 5120.00, iops: 3.99, rt: 511.25) [2024-08-16 10:19:56.540156] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:56.540177] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540187] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540193] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540205] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=11] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540212] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540222] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540227] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540232] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540236] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540242] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540252] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540257] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540261] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540271] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540276] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540282] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:56.540340] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [31787][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=17] try gc retain ctx [2024-08-16 10:19:56.540744] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:56.540767] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:56.540775] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:56.540780] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:56.542275] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=28][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:56.548529] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.548551] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:56.548561] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.548573] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.548587] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.549888] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A69-0-0] [lt=45] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:19:56.550633] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=554, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:19:56.550670] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=32, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:19:56.550744] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=35, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:19:56.556766] INFO [SHARE] refresh_primary_cluster_id (ob_alive_server_tracer.cpp:397) [31391][ServerTracerTim][T0][YB42C0A80080-00061CCC8EABC3A1-0-0] [lt=23] refresh primary cluster finish(ret=0, primary_cluster_id_=-1) [2024-08-16 10:19:56.560537] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1501) [2024-08-16 10:19:56.564572] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [31733][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=19] run rewrite rule refresh task(rule_mgr_->tenant_id_=1) [2024-08-16 10:19:56.590791] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=9][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:19:56.593145] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=31] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:19:56.593254] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=25][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:19:56.593284] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=28][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:19:56.593295] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=9] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:19:56.596928] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.596950] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:56.596961] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.596977] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.596999] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.606624] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=23] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:19:56.610461] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=25] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:19:56.616540] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=23] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60666, mem_used=137263104) [2024-08-16 10:19:56.627220] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=35] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=2579474, replace_num=31457) [2024-08-16 10:19:56.627244] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=22] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=10) [2024-08-16 10:19:56.640848] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:56.640965] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:56.640987] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:56.640995] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:56.642172] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73536-0-0] [lt=31] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=5, avg_log_gen_to_submit_time=150, avg_log_submit_to_flush_time=671, avg_log_submit_to_slide_time=1172) [2024-08-16 10:19:56.642377] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=29][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:56.645331] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [31772][T1_Flush][T1][Y0-0000000000000000-0-0] [lt=22] ====== traversal_flush timer task ====== [2024-08-16 10:19:56.645352] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:338) [31772][T1_Flush][T1][Y0-0000000000000000-0-0] [lt=16] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:19:56.648597] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.648619] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:56.648632] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.648643] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.648669] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.651017] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73537-0-0] [lt=23] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:19:56.651048] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73537-0-0] [lt=30] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1077, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:19:56.697020] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.697042] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:56.697056] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.697068] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.697119] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=32] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.712492] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:56.712766] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=28] Wash time detail, (compute_wash_size_time=129, refresh_score_time=84, wash_time=15) [2024-08-16 10:19:56.715974] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=15] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=503312, clean_num=62914) [2024-08-16 10:19:56.732537] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=12] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1001) [2024-08-16 10:19:56.732566] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=25] start lib cache evict(tenant_id=1001, mem_hold=48234496, mem_limit=53687090, cache_obj_num=115, cache_node_num=115) [2024-08-16 10:19:56.732576] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=9] end lib cache evict(tenant_id=1001, cache_evict_num=0, mem_hold=48234496, mem_limit=53687090, cache_obj_num=115, cache_node_num=115) [2024-08-16 10:19:56.732583] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=6] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:19:56.735536] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=5] Dumping All Cache Objs(alloc_obj_list.count()=115, alloc_obj_list=[{obj_id:4206, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190714}, {obj_id:6967, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:7888, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190463}, {obj_id:8067, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198580}, {obj_id:8069, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179360}, {obj_id:8462, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:252321}, {obj_id:8631, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176478}, {obj_id:8633, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:8641, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188033}, {obj_id:8645, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8646, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:8647, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8648, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:8649, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8651, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8652, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8653, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188613}, {obj_id:8682, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187574}, {obj_id:8692, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234101}, {obj_id:8697, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227212}, {obj_id:8767, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245433}, {obj_id:8786, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279819}, {obj_id:8795, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:281759}, {obj_id:8805, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191548}, {obj_id:8840, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:293038}, {obj_id:8842, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:324921}, {obj_id:8843, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:456542}, {obj_id:8851, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:575647}, {obj_id:8854, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:347535}, {obj_id:8873, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:452546}, {obj_id:8878, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:582753}, {obj_id:8885, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:474728}, {obj_id:8886, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:329638}, {obj_id:8891, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:452691}, {obj_id:8892, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:375165}, {obj_id:8896, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:356761}, {obj_id:8897, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290927}, {obj_id:8898, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:413784}, {obj_id:8906, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8907, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8908, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189807}, {obj_id:8909, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189568}, {obj_id:8910, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176333}, {obj_id:8912, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196245}, {obj_id:8913, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8914, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180998}, {obj_id:8915, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174064}, {obj_id:8916, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177216}, {obj_id:8917, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:8918, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196176}, {obj_id:8919, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177512}, {obj_id:8920, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8921, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:8922, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:8923, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180749}, {obj_id:8925, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:253523}, {obj_id:8926, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199084}, {obj_id:8927, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178773}, {obj_id:8928, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177464}, {obj_id:8929, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179416}, {obj_id:8930, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180672}, {obj_id:8931, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8932, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186657}, {obj_id:8933, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233944}, {obj_id:8934, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185434}, {obj_id:8935, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198144}, {obj_id:8936, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207413}, {obj_id:8937, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189999}, {obj_id:8938, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188514}, {obj_id:8939, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188880}, {obj_id:8940, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231829}, {obj_id:8941, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190949}, {obj_id:8942, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192036}, {obj_id:8943, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187766}, {obj_id:8944, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195808}, {obj_id:8946, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176485}, {obj_id:8947, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:386890}, {obj_id:8948, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8949, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202973}, {obj_id:8950, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183752}, {obj_id:8951, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223501}, {obj_id:8952, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188501}, {obj_id:8953, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186993}, {obj_id:8954, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:8955, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:8956, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:8957, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176676}, {obj_id:8958, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175405}, {obj_id:8959, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199144}, {obj_id:8960, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205144}, {obj_id:8961, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218139}, {obj_id:8962, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218827}, {obj_id:8963, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217152}, {obj_id:8964, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218674}, {obj_id:8965, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8966, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206801}, {obj_id:8967, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256033}, {obj_id:8968, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8969, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199798}, {obj_id:8970, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199390}, {obj_id:8971, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220936}, {obj_id:8972, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209427}, {obj_id:8973, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220503}, {obj_id:8974, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221262}, {obj_id:8975, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189045}, {obj_id:8976, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194445}, {obj_id:8977, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189493}, {obj_id:8978, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196989}, {obj_id:8979, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190072}, {obj_id:8980, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202973}, {obj_id:8981, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181806}, {obj_id:8982, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180351}, {obj_id:8983, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177512}, {obj_id:8984, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188650}, {obj_id:8985, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179512}]) [2024-08-16 10:19:56.735834] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=294] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:19:56.740926] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:56.741042] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=129, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:19:56.741109] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT APPEND COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:19:56.741129] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:56.741142] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:56.741150] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:56.742424] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:56.748660] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.748682] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:56.748692] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.748702] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.748716] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.759904] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFB7-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:19:56.759950] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFB7-0-0] [lt=24] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=9, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:19:56.759964] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFB7-0-0] [lt=13] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=111230, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:19:56.759975] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFB7-0-0] [lt=9][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509817, log_task={header:{begin_lsn:{lsn:18544215995}, end_lsn:{lsn:18544216117}, log_id:53509817, min_scn:{val:1723774796573378000, v:0}, max_scn:{val:1723774796573378000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544215873}, prev_proposal_id:7, committed_end_lsn:{lsn:18544215995}, data_checksum:1913811101, accum_checksum:1316606007, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774796659687, freeze_ts:1723774796659687, submit_ts:1723774796659692, flushed_ts:1723774796660403, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:711}, fs_cb_begin_ts=1723774796759940, log_life_time=100253) [2024-08-16 10:19:56.797109] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.797136] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:19:56.797148] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.797159] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.797173] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.827338] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=10] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=9) [2024-08-16 10:19:56.841000] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:56.841131] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:56.841148] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:56.841155] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:56.841754] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1782) [2024-08-16 10:19:56.842497] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:56.848742] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.848764] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:56.848774] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.848783] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.848794] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.858408] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [2421][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=53] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,1,False,{False,},False,False,True}, {1024,1,False,{False,},False,False,True}, {1026,1,False,{False,},False,False,True}, {1028,0,False,{False,},False,False,True}, {1079,1,False,{False,},False,False,True}, {1260,0,False,{False,},False,False,True}, {1261,1,False,{False,},False,False,True}, {1262,1,False,{False,},False,False,True}]) [2024-08-16 10:19:56.860423] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=626, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:19:56.874378] INFO [PALF] read (log_cache.cpp:100) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:56.897196] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.897243] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=45] handle task succ(id={id:1260}) [2024-08-16 10:19:56.897255] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.897264] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.897279] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.916449] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:56.916750] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=55] Wash time detail, (compute_wash_size_time=144, refresh_score_time=65, wash_time=26) [2024-08-16 10:19:56.917642] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=17] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:19:56.919974] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=566226, clean_num=62914) [2024-08-16 10:19:56.941058] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:56.941197] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=48] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=12, avg_log_batch_cnt=1, total_group_log_size=1904, avg_group_log_size=158) [2024-08-16 10:19:56.941227] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1670) [2024-08-16 10:19:56.941242] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:56.941255] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:56.941262] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:56.942590] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:56.948823] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.948846] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:19:56.948856] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.948886] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=28] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.948901] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:19:56.949799] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=44] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:56.949824] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=18] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:56.950106] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:19:56.950127] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=15][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:19:56.950136] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:19:56.950144] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:19:56.951253] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74332-0-0] [lt=45] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=9, avg_log_submit_to_flush_time=610, avg_log_submit_to_slide_time=1047) [2024-08-16 10:19:56.951282] INFO [PALF] read (log_cache.cpp:100) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=75] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:56.960144] INFO [STORAGE.DUP_TABLE] follower_try_acquire_lease (ob_dup_table_lease.cpp:500) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=27] [DupLeaseLife] The follower can get new lease from this lease log(ret=0, ls_id={id:1079}, lease_log_scn={val:1723774796573378003, v:0}, self_addr="192.168.0.128:2882", acquire_new_lease=false, follower_lease_info_={durable_lease_:{request_ts_:1723774796541195, lease_interval_us_:6000000}, lease_expired_ts_:1723774802541195, last_lease_scn_:{val:1723774796573378003, v:0}, lease_acquire_scn_:{val:1722500589629865001, v:0}}) [2024-08-16 10:19:56.974643] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74334-0-0] [lt=21] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=647, avg_log_submit_to_slide_time=1073) [2024-08-16 10:19:56.997280] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:56.997305] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:19:56.997317] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:56.997333] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:56.997355] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.011966] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=24] gc stale ls task succ [2024-08-16 10:19:57.012093] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=14] gc stale ls task succ [2024-08-16 10:19:57.012124] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=20] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:19:57.012145] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=15] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:19:57.012153] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=6] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:19:57.016608] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=19] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:57.016608] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:57.022876] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C02-0-0] [lt=25] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774797022831) [2024-08-16 10:19:57.027449] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=30] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=8) [2024-08-16 10:19:57.034059] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1723774797034057, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007613, add_bytes: 0 [2024-08-16 10:19:57.041103] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:57.041227] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:57.041246] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:57.041253] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:57.041880] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1670) [2024-08-16 10:19:57.042667] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:57.048877] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.048901] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:19:57.048911] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.048920] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.048944] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.049713] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=22] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:19:57.050911] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1342) [2024-08-16 10:19:57.056359] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2747][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=22] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:19:57.060225] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFC7-0-0] [lt=40][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509821, log_task={header:{begin_lsn:{lsn:18544216764}, end_lsn:{lsn:18544216886}, log_id:53509821, min_scn:{val:1723774796874628000, v:0}, max_scn:{val:1723774796874628000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544216361}, prev_proposal_id:7, committed_end_lsn:{lsn:18544216764}, data_checksum:1913811101, accum_checksum:946261648, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774796959946, freeze_ts:1723774796959946, submit_ts:1723774796959954, flushed_ts:1723774796960650, gen_to_freeze cost time:0, gen_to_submit cost time:8, submit_to_flush cost time:696}, fs_cb_begin_ts=1723774797060209, log_life_time=100263) [2024-08-16 10:19:57.060324] INFO [LIB] stat (utility.h:1197) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=30] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=39, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:19:57.060348] INFO [LIB] stat (utility.h:1197) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=23] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=8, this=0x7fa713668b28, extra_info="") [2024-08-16 10:19:57.061059] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=34] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=37, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:19:57.064646] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:19:57.064672] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:19:57.073265] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=29] PNIO [ratelimit] time: 1723774797073263, bytes: 1496435899699, bw: 0.115612 MB/s, add_ts: 1000486, add_bytes: 121287 [2024-08-16 10:19:57.073717] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85AC-0-0] [lt=60] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:19:57.097305] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:19:57.097372] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.097402] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:19:57.097416] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.097443] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.097466] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.100809] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AAA-0-0] [lt=51] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:19:57.100858] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AAA-0-0] [lt=31] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774796752453002, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774796752453002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:19:57.110205] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAD4-0-0] [lt=31] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11027, add_timestamp:1723774797110186}) [2024-08-16 10:19:57.110250] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766EF-0-0] [lt=69] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11027, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:19:57.110365] INFO [SERVER] fill_tenant_ids_ (ob_all_virtual_tx_stat.cpp:139) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAD4-0-0] [lt=21] succeed to get tenant ids(all_tenants_=[1, 1001, 1002]) [2024-08-16 10:19:57.110482] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [21121][T1_L0_G0][T1][YB42C0A80085-00061CCD1E636C11-0-0] [lt=27][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:57.110718] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766EF-0-0] [lt=24] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11027, locations=[{table_id:11027, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:19:57.110777] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766EF-0-0] [lt=58] success to process renew task(task={tenant_id:1, table_id:11027, add_timestamp:1723774797110186}, locations=[{table_id:11027, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774797110689, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:19:57.118110] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7D7-0-0] [lt=77] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:19:57.120446] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:57.120724] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=44] Wash time detail, (compute_wash_size_time=147, refresh_score_time=74, wash_time=16) [2024-08-16 10:19:57.123942] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=629140, clean_num=62914) [2024-08-16 10:19:57.141174] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:57.141282] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=29] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:57.141298] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:57.141305] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:57.141669] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=38] get_min_unreplayed_log_info(lsn={lsn:18544216886}, scn={val:1723774796874628001, v:0}, this={ls_id_:{id:1079}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774797060211, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:11904, ret_code_:0}}, next_to_submit_lsn_:{lsn:18544216886}, next_to_submit_scn_:{val:1723774796874628001, v:0}, base_lsn_:{lsn:13018324992}, base_scn_:{val:1720509584063969004, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd005000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:18544216764}, end_lsn:{lsn:18544216886}, read_buf:{buf_len_:2154496, buf_:0x7fa6cd005000}, block_size:67104768, log_storage_:0x7fa6fc6ef470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774796874628000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774796874628000, v:0}, data_checksum:959419206, flag:0}}, init_mode_version:0, accumulate_checksum:946261648, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774796874628000, v:0}}}}}) [2024-08-16 10:19:57.142742] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:134) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22] ObStandbyTimestampService state(*this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:19:57.142767] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:57.148955] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.148973] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] handle task succ(id={id:1260}) [2024-08-16 10:19:57.148982] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.148992] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.149004] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.150382] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A75-0-0] [lt=23] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:19:57.150407] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A75-0-0] [lt=23] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:19:57.157233] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF_DUMP](palf_id=1260, self="192.168.0.128:2882", [SlidingWindow]={palf_id:1260, self:"192.168.0.128:2882", lsn_allocator:{max_log_id:34644041, max_lsn:8828135178, max_scn:{val:1723774796916390002, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:8828135178}, reserved_buffer_size:41943040, available_buffer_size:33554432, readable_begin_lsn:{lsn:8786192138}}, last_submit_lsn:{lsn:8828135056}, last_submit_end_lsn:{lsn:8828135178}, last_submit_log_id:34644041, last_submit_log_pid:29, max_flushed_lsn:{lsn:8828135056}, max_flushed_end_lsn:{lsn:8828135178}, max_flushed_log_pid:29, committed_end_lsn:{lsn:8828135178}, last_slide_log_id:34644041, last_slide_scn:{val:1723774796916390002, v:0}, last_slide_lsn:{lsn:8828135056}, last_slide_end_lsn:{lsn:8828135178}, last_slide_log_pid:29, last_slide_log_accum_checksum:3425551535, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x7fa6bd554530}, [StateMgr]={this:0x7fa6bd5582b0, self:"192.168.0.128:2882", palf_id:1260, role:"LEADER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:29}, leader:"192.168.0.128:2882", leader_epoch:32, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:8469210191}, scan_disk_log_finished:true, last_check_start_id:34644042, is_changing_config_with_arb:false, reconfirm_start_time_us:1723617929285618, palf_role_change_cb:0x7fa6bd55eb70, allow_vote:true}, [ConfigMgr]={palf_id:1260, self:"192.168.0.128:2882", alive_paxos_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, alive_paxos_replica_num:3, log_ms_meta:{version:3, proposal_id:29, prev:{version:1, config:{config_version:{proposal_id:29, config_seq:443}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:29240, lock_time:1723646523720109}}, curr:{version:1, config:{config_version:{proposal_id:29, config_seq:444}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29240, lock_time:1723646524240102}}, prev_log_proposal_id:29, prev_lsn:{lsn:8514860172}, prev_mode_pid:29}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:29240, lock_type:0, added_list:0, removed_list:0, type:"UNLOCK_CONFIG_CHANGE"}, state:0, checking_barrier:{prev_log_proposal_id:29, prev_lsn:{lsn:8514860172}, prev_end_lsn:{lsn:8514860294}, prev_mode_pid:29}, reconfig_barrier:{prev_log_proposal_id:29, prev_lsn:{lsn:8514860172}, prev_end_lsn:{lsn:8514860294}, prev_mode_pid:29}, persistent_config_version:{proposal_id:29, config_seq:444}, ms_ack_list:0, resend_config_version:{proposal_id:29, config_seq:444}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1723617925582820, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1723617925582820, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:1722595738376965, this:0x7fa6bd555c70}, [ModeMgr]={palf_id:1260, self:"192.168.0.128:2882", applied_mode_meta:{version:1, proposal_id:29, mode_version:0, access_mode:1, ref_scn:{val:1720516541172103000, v:0}}, accepted_mode_meta:{version:1, proposal_id:29, mode_version:0, access_mode:1, ref_scn:{val:1720516541172103000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:29, mode_version:0, access_mode:1, ref_scn:{val:1720516541172103000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:1723617929485855, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1260, is_inited:true, min_block_max_scn:{val:1721971839646833003, v:0}, min_block_id:75, min_block_min_scn:{val:1721937225099170005, v:0}, base_lsn_for_block_gc:{lsn:8723619840}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:29}, log_config_meta:{version:3, proposal_id:29, prev:{version:1, config:{config_version:{proposal_id:29, config_seq:443}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:29240, lock_time:1723646523720109}}, curr:{version:1, config:{config_version:{proposal_id:29, config_seq:444}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29240, lock_time:1723646524240102}}, prev_log_proposal_id:29, prev_lsn:{lsn:8514860172}, prev_mode_pid:29}, log_snapshot_meta:{version:1, base_lsn:{lsn:8723619840}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:29, mode_version:0, access_mode:1, ref_scn:{val:1720516541172103000, v:0}}}, log_meta_storage:{log_tail:{lsn:1409024}, readable_log_tail:{lsn:1409024}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:8857829376}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1260}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1260/meta", dir_fd:192, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:65695744, block_header_serialize_buf_:0x7fa6bd55a438, flashback_version:0}, log_storage:{log_tail:{lsn:8828135178}, readable_log_tail:{lsn:8828135178}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:8790724608}, min_scn:{val:1723749999379189001, v:0}, curr_block_id:131, palf_id:1260}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1260/log", dir_fd:196, min_block_id:75, max_block_id:132, curr_writable_block_id:131}, logical_block_size_:67104768, curr_block_writable_size_:29694198, block_header_serialize_buf_:0x7fa6bd55c0b8, flashback_version:0}, palf_epoch:2, last_purge_throttling_ts:1723617929284852, this:0x7fa6bd5589f0}, [Reconfirm]={palf_id:1260, self:"192.168.0.128:2882", state:"FINISHED", new_proposal_id:29, prepare_log_ack_list:2"192.168.0.125:2882";"192.168.0.133:2882";, curr_paxos_follower_list:2{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, majority_cnt:2, majority_max_log_server:"192.168.0.128:2882", majority_max_accept_pid:28, majority_max_lsn:{lsn:8469424174}, saved_end_lsn:{lsn:8469424174}, last_submit_prepare_req_time_us:1723617929285921, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x7fa6bd558430}) [2024-08-16 10:19:57.157434] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=196] [PALF_DUMP](palf_id=1260, self="192.168.0.128:2882", ack_info_list=[{member:{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}, last_ack_time_us:1723774797142430, last_flushed_end_lsn:{lsn:8828135178}}, {member:{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}, last_ack_time_us:1723774797141999, last_flushed_end_lsn:{lsn:8828135178}}, {member:{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, last_ack_time_us:1723774797142358, last_flushed_end_lsn:{lsn:8828135178}}]) [2024-08-16 10:19:57.157452] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=16] current log_task status(palf_id=1260, self="192.168.0.128:2882", log_id=34644042, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[{member:{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}, last_ack_time_us:1723774797142430, last_flushed_end_lsn:{lsn:8828135178}}, {member:{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}, last_ack_time_us:1723774797141999, last_flushed_end_lsn:{lsn:8828135178}}, {member:{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, last_ack_time_us:1723774797142358, last_flushed_end_lsn:{lsn:8828135178}}]) [2024-08-16 10:19:57.169312] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAD5-0-0] [lt=25][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=370, pos=300, bitmap=NULL) [2024-08-16 10:19:57.173640] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85AE-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:19:57.183226] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=29] gc stale ls task succ [2024-08-16 10:19:57.184047] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCC998AE0A6-0-0] [lt=19] receive register_req from acceptor, ignore(self="192.168.0.128:2882", palf_id=1260, child={server:"192.168.0.125:2882", region:"sys_region", register_time_us:1723774797183009, keepalive_ts:-1}) [2024-08-16 10:19:57.184091] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCC998AE0A6-0-0] [lt=42] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1260, self="192.168.0.128:2882", child={server:"192.168.0.125:2882", region:"sys_region", register_time_us:1723774797183009, keepalive_ts:-1}, is_to_leader=true) [2024-08-16 10:19:57.197440] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.197469] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=26] handle task succ(id={id:1260}) [2024-08-16 10:19:57.197480] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.197489] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.197505] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.198958] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2520][T1002_TenantMet][T1002][Y0-0000000000000000-0-0] [lt=45] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:17374, free_obj_hold_cnt:593, allocator used:71436792, allocator total:73453184}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:1994, free_obj_hold_cnt:6, allocator used:131182000, allocator total:131182000}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():63488, full allocator used:1600, full allocator total:129024}, tablets_mem=204764208, tablets_mem_limit=1288490180, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:7, allocator used:17024, allocator total:65408}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:106, free_obj_hold_cnt:5057, allocator used:10243392, allocator total:10596096}, pending_cnt=0, wait_gc_count=0, tablet count=19368) [2024-08-16 10:19:57.212935] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE7C-0-0] [lt=27] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:19:57.212975] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE7C-0-0] [lt=25] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774796857494003, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774796857494003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:19:57.225008] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [2777][T1001_ObXAHbWor][T1001][Y0-0000000000000000-0-0] [lt=34] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-08-16 10:19:57.227557] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=23] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=7) [2024-08-16 10:19:57.231047] INFO [CLOG] is_in_sync (ob_log_handler.cpp:546) [31817][T1_L5_G0][T1002][YB42C0A8007E-00061EFF757F6EAC-0-0] [lt=27] is_in_sync(ret=0, id=1079, is_log_sync=true, is_need_rebuild=false) [2024-08-16 10:19:57.231140] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007E-00061EFF757F6EAC-0-0] [lt=22][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:57.235740] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=21] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:57.236482] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=23] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:57.236853] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=17] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:57.237120] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=14] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:57.237217] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:57.237225] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:57.237268] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=30] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:57.238963] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD16DAE252-0-0] [lt=13] receive register_req from acceptor, ignore(self="192.168.0.128:2882", palf_id=1260, child={server:"192.168.0.133:2882", region:"sys_region", register_time_us:1723774797236700, keepalive_ts:-1}) [2024-08-16 10:19:57.238989] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD16DAE252-0-0] [lt=25] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1260, self="192.168.0.128:2882", child={server:"192.168.0.133:2882", region:"sys_region", register_time_us:1723774797236700, keepalive_ts:-1}, is_to_leader=true) [2024-08-16 10:19:57.241250] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:57.241373] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:57.241396] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:57.241403] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:57.242058] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=600, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:19:57.242131] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=58] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=26, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:19:57.242159] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=43] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:19:57.242596] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE57C-0-0] [lt=13] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:19:57.242613] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE57C-0-0] [lt=16] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1237, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:19:57.242849] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=10][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:57.244186] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=47] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574612, offset=42140413}, end_cursor:ObLogCursor{file_id=65, log_id=18574613, offset=42140654}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751614050, len:3331, log_data_len:241, seq:18574612, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:57.244975] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=51] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574613, offset=42140654}, end_cursor:ObLogCursor{file_id=65, log_id=18574614, offset=42140903}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75163c050, len:3090, log_data_len:249, seq:18574613, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:57.245133] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80086-00061EFF757FAF11-0-0] [lt=16][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:57.246474] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BC9F-0-0] [lt=19] push gts task success(*task={:""}) [2024-08-16 10:19:57.249037] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.249056] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] handle task succ(id={id:1260}) [2024-08-16 10:19:57.249066] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.249095] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=27] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.249112] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.256804] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BC9F-0-0] [lt=17] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:19:57.259169] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=27] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574614, offset=42140903}, end_cursor:ObLogCursor{file_id=65, log_id=18574615, offset=42141152}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75164c050, len:2841, log_data_len:249, seq:18574614, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:57.259755] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=37] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574615, offset=42141152}, end_cursor:ObLogCursor{file_id=65, log_id=18574616, offset=42141401}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516d4050, len:2592, log_data_len:249, seq:18574615, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:57.260278] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFD1-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1623, avg_group_log_size=147) [2024-08-16 10:19:57.260329] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFD1-0-0] [lt=31][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509823, log_task={header:{begin_lsn:{lsn:18544217008}, end_lsn:{lsn:18544217130}, log_id:53509823, min_scn:{val:1723774797151134000, v:0}, max_scn:{val:1723774797151134000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544216886}, prev_proposal_id:7, committed_end_lsn:{lsn:18544217008}, data_checksum:1913811101, accum_checksum:3574457637, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774797160150, freeze_ts:1723774797160150, submit_ts:1723774797160155, flushed_ts:1723774797160763, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:608}, fs_cb_begin_ts=1723774797260316, log_life_time=100166) [2024-08-16 10:19:57.270377] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=24] current monitor number(seq_=-1) [2024-08-16 10:19:57.273812] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85B0-0-0] [lt=94] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1342, avg_group_log_size=122) [2024-08-16 10:19:57.274463] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=580, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:19:57.274498] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:19:57.274535] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=34, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:19:57.274608] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=34, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:19:57.274940] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE57E-0-0] [lt=30] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:19:57.274959] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE57E-0-0] [lt=18] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1053, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:19:57.297481] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.297504] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:19:57.297516] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.297527] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.297541] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.299169] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=11] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:19:57.299199] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=26] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=4982, mem_used=39515136) [2024-08-16 10:19:57.324422] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:57.324697] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=44] Wash time detail, (compute_wash_size_time=150, refresh_score_time=67, wash_time=16) [2024-08-16 10:19:57.327754] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=692054, clean_num=62914) [2024-08-16 10:19:57.341310] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:57.341428] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:57.341445] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:57.341452] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:57.342935] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=24][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:57.349108] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.349129] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:57.349141] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.349151] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.349166] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.360505] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFD7-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1623) [2024-08-16 10:19:57.360549] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFD7-0-0] [lt=27] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=707, avg_log_submit_to_slide_time=91000) [2024-08-16 10:19:57.360594] INFO [PALF] read (log_cache.cpp:100) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1623, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:57.383677] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=20] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:19:57.397558] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.397582] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:19:57.397594] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.397605] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.397620] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.424854] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=19] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:19:57.424950] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=18] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:19:57.427669] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=22] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=6) [2024-08-16 10:19:57.441375] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:57.441482] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:57.441511] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:57.441519] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:57.443011] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:57.444875] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [31779][T1_ArcSender][T1][YB42C0A80080-00061CCC8D41D0C5-0-0] [lt=24][errcode=-4018] try free send task failed(ret=-4018) [2024-08-16 10:19:57.449176] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.449197] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:57.449208] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.449218] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.449232] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.460334] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [2790][T1001_ArcSender][T1001][YB42C0A80080-00061CCC9351D0C5-0-0] [lt=21][errcode=-4018] try free send task failed(ret=-4018) [2024-08-16 10:19:57.473723] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [2517][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=22] run rewrite rule refresh task(rule_mgr_->tenant_id_=1002) [2024-08-16 10:19:57.485638] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [2672][T1001_PalfGC][T1001][Y0-0000000000000000-0-0] [lt=29] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):2150, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):2150, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}) [2024-08-16 10:19:57.486478] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7DC-0-0] [lt=26] [RS_MGR] force set rs list(master_rs="192.168.0.126:2882") [2024-08-16 10:19:57.486508] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7DC-0-0] [lt=22] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"192.168.0.128:2882", server_id:16, rs_addr:"192.168.0.126:2882", rs_server_status:1, epoch_id:11}, hb_response={zone:"zone2", server:"192.168.0.128:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1720514727760575, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=30) [2024-08-16 10:19:57.497635] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.497657] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:57.497670] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.497681] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.497709] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.525118] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=26] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=6878, mem_used=12478464) [2024-08-16 10:19:57.528231] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31502][AutoLSLoc][T0][YB42C0A80080-00061CCC8EEBBF6B-0-0] [lt=93] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:19:57.529179] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:57.529453] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=1087, refresh_score_time=79, wash_time=22) [2024-08-16 10:19:57.532545] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=24] start do ls ha handler(ls_id_array_=[{id:1260}, {id:1262}, {id:1079}, {id:1261}]) [2024-08-16 10:19:57.532634] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=754968, clean_num=62914) [2024-08-16 10:19:57.536919] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [31638][T1_Occam][T1][YB42C0A80080-00061CCCBF6D3E08-0-0] [lt=30] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:19:57.540387] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=9] trace max work area(auto_calc=true, tenant_max_memory_limit=1073741824, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=53687091, max_wa_memory_size=53687091, tmp_max_wa_memory_size=53687091, pre_mem_target=53687091, remain_memory_size=53687091, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=332681216, washable_size=-2, max_workarea_memory_size=53687091, max_tenant_memory_size=741060608, wash_ratio=6, tenant_id=1001) [2024-08-16 10:19:57.540513] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=38] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=6710886, calc_global_bound_size=1048576000000, mem_target_=53687091, wa_max_memory_size=53687091, profile_cnt_=0, total_memory_size=0, max_wa_size=53687091, avg_bound_size=6710886, max_bound_size=6710886, min_bound_size_=1048576) [2024-08-16 10:19:57.540555] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=38] timer to calc global bound size(ret=0, global_bound_size_=6710886, manual_calc_cnt_=0, drift_size_=0, pre_drift_size=0, wa_max_memory_size=53687091, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1001, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=6710886, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=53687091, auto_calc=true, sql_mem_callback_.get_total_dump_size()=0) [2024-08-16 10:19:57.540631] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=34] trace max work area(auto_calc=true, tenant_max_memory_limit=6442450944, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=3865470566, max_wa_memory_size=3865470566, tmp_max_wa_memory_size=3865470566, pre_mem_target=3865470566, remain_memory_size=3865470566, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=2461601792, washable_size=-2, max_workarea_memory_size=3865470566, max_tenant_memory_size=3980849152, wash_ratio=6, tenant_id=1002) [2024-08-16 10:19:57.540700] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=25] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=483183820, calc_global_bound_size=1048576000000, mem_target_=3865470566, wa_max_memory_size=3865470566, profile_cnt_=0, total_memory_size=0, max_wa_size=3865470566, avg_bound_size=483183820, max_bound_size=483183820, min_bound_size_=1048576) [2024-08-16 10:19:57.540720] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=18] timer to calc global bound size(ret=0, global_bound_size_=483183820, manual_calc_cnt_=19637, drift_size_=0, pre_drift_size=0, wa_max_memory_size=3865470566, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1002, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=483183820, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=3865470566, auto_calc=true, sql_mem_callback_.get_total_dump_size()=1112786688) [2024-08-16 10:19:57.541449] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:57.541541] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:57.541567] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:57.541576] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:57.543096] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:57.543265] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 321.00) [2024-08-16 10:19:57.543291] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=21] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:57.543365] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=39] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.99, rt: 478.25) [2024-08-16 10:19:57.543379] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=11] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:57.543401] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=16] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543415] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=12] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543422] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543429] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543438] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543444] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543454] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543459] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543463] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543470] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543479] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543486] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543496] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543513] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543521] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.543527] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:57.549249] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.549270] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:57.549280] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.549289] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.549303] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.550665] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A7D-0-0] [lt=12] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:19:57.551305] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=44] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=546, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:19:57.551336] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=21, stat_interval=1000000, avg (count/cost)=34, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:19:57.551392] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=34, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:19:57.557559] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=31] [PALF_DUMP](palf_id=1079, self="192.168.0.128:2882", [SlidingWindow]={palf_id:1079, self:"192.168.0.128:2882", lsn_allocator:{max_log_id:53509826, max_lsn:18544217496, max_scn:{val:1723774797375521000, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:18544217496}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:18502274456}}, last_submit_lsn:{lsn:18544217374}, last_submit_end_lsn:{lsn:18544217496}, last_submit_log_id:53509826, last_submit_log_pid:7, max_flushed_lsn:{lsn:18544217374}, max_flushed_end_lsn:{lsn:18544217496}, max_flushed_log_pid:7, committed_end_lsn:{lsn:18544217374}, last_slide_log_id:53509825, last_slide_scn:{val:1723774797151134002, v:0}, last_slide_lsn:{lsn:18544217252}, last_slide_end_lsn:{lsn:18544217374}, last_slide_log_pid:7, last_slide_log_accum_checksum:3847999696, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x7fa6fc6e8530}, [StateMgr]={this:0x7fa6fc6ec2b0, self:"192.168.0.128:2882", palf_id:1079, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:7}, leader:"0.0.0.0:0", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:53509826, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x7fa6fc6f2b70, allow_vote:true}, [ConfigMgr]={palf_id:1079, self:"192.168.0.128:2882", alive_paxos_memberlist:3{server:"192.168.0.124:2882", timestamp:1, flag:0}{server:"192.168.0.127:2882", timestamp:1, flag:0}{server:"192.168.0.131:2882", timestamp:1, flag:0}, alive_paxos_replica_num:3, log_ms_meta:{version:3, proposal_id:7, prev:{version:1, config:{config_version:{proposal_id:7, config_seq:37}, log_sync_memberlist:3{server:"192.168.0.124:2882", timestamp:1, flag:0}{server:"192.168.0.127:2882", timestamp:1, flag:0}{server:"192.168.0.131:2882", timestamp:1, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:4, learner_array:[{server:"192.168.0.125:2882", timestamp:1720516537043883, flag:0}, {server:"192.168.0.126:2882", timestamp:1720513938601680, flag:0}, {server:"192.168.0.128:2882", timestamp:1720516609055334, flag:0}, {server:"192.168.0.133:2882", timestamp:1720516557506171, flag:0}]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:7, config_seq:38}, log_sync_memberlist:3{server:"192.168.0.124:2882", timestamp:1, flag:0}{server:"192.168.0.127:2882", timestamp:1, flag:0}{server:"192.168.0.131:2882", timestamp:1, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:3, learner_array:[{server:"192.168.0.125:2882", timestamp:1720516537043883, flag:0}, {server:"192.168.0.128:2882", timestamp:1720516609055334, flag:0}, {server:"192.168.0.133:2882", timestamp:1720516557506171, flag:0}]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:7, prev_lsn:{lsn:16645750544}, prev_mode_pid:7}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:7, config_seq:38}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1720516633177815, parent:"192.168.0.131:2882", parent_keepalive_time_us:1723774797022831, last_submit_register_req_time_us:-1, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x7fa6fc6e9c70}, [ModeMgr]={palf_id:1079, self:"192.168.0.128:2882", applied_mode_meta:{version:1, proposal_id:7, mode_version:0, access_mode:1, ref_scn:{val:1718350230743746000, v:0}}, accepted_mode_meta:{version:1, proposal_id:7, mode_version:0, access_mode:1, ref_scn:{val:1718350230743746000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:7, mode_version:0, access_mode:1, ref_scn:{val:1718350230743746000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1079, is_inited:true, min_block_max_scn:{val:1721964469082002002, v:0}, min_block_id:226, min_block_min_scn:{val:1721914383906628001, v:0}, base_lsn_for_block_gc:{lsn:18453811200}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:7}, log_config_meta:{version:3, proposal_id:7, prev:{version:1, config:{config_version:{proposal_id:7, config_seq:37}, log_sync_memberlist:3{server:"192.168.0.124:2882", timestamp:1, flag:0}{server:"192.168.0.127:2882", timestamp:1, flag:0}{server:"192.168.0.131:2882", timestamp:1, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:4, learner_array:[{server:"192.168.0.125:2882", timestamp:1720516537043883, flag:0}, {server:"192.168.0.126:2882", timestamp:1720513938601680, flag:0}, {server:"192.168.0.128:2882", timestamp:1720516609055334, flag:0}, {server:"192.168.0.133:2882", timestamp:1720516557506171, flag:0}]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:7, config_seq:38}, log_sync_memberlist:3{server:"192.168.0.124:2882", timestamp:1, flag:0}{server:"192.168.0.127:2882", timestamp:1, flag:0}{server:"192.168.0.131:2882", timestamp:1, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:3, learner_array:[{server:"192.168.0.125:2882", timestamp:1720516537043883, flag:0}, {server:"192.168.0.128:2882", timestamp:1720516609055334, flag:0}, {server:"192.168.0.133:2882", timestamp:1720516557506171, flag:0}]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:7, prev_lsn:{lsn:16645750544}, prev_mode_pid:7}, log_snapshot_meta:{version:1, base_lsn:{lsn:18453811200}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:7, mode_version:0, access_mode:1, ref_scn:{val:1718350230743746000, v:0}}}, log_meta_storage:{log_tail:{lsn:225280}, readable_log_tail:{lsn:225280}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:18588020736}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1079}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1079/meta", dir_fd:148, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:66879488, block_header_serialize_buf_:0x7fa6fc6ee438, flashback_version:0}, log_storage:{log_tail:{lsn:18544217496}, readable_log_tail:{lsn:18544217496}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:18520915968}, min_scn:{val:1723759521816553001, v:0}, curr_block_id:276, palf_id:1079}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1079/log", dir_fd:156, min_block_id:226, max_block_id:277, curr_writable_block_id:276}, logical_block_size_:67104768, curr_block_writable_size_:43803240, block_header_serialize_buf_:0x7fa6fc6f00b8, flashback_version:0}, palf_epoch:1, last_purge_throttling_ts:-1, this:0x7fa6fc6ec9f0}, [Reconfirm]={palf_id:1079, self:"192.168.0.128:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x7fa6fc6ec430}) [2024-08-16 10:19:57.557774] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=210][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1079, self="192.168.0.128:2882", tmp_server={server:"192.168.0.124:2882", timestamp:1, flag:0}) [2024-08-16 10:19:57.557789] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=14] [PALF_DUMP](palf_id=1079, self="192.168.0.128:2882", ack_info_list=[]) [2024-08-16 10:19:57.557799] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1079, self="192.168.0.128:2882", tmp_server={server:"192.168.0.124:2882", timestamp:1, flag:0}) [2024-08-16 10:19:57.557806] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=6] current log_task status(palf_id=1079, self="192.168.0.128:2882", log_id=53509826, log_task={header:{begin_lsn:{lsn:18544217374}, end_lsn:{lsn:18544217496}, log_id:53509826, min_scn:{val:1723774797375521000, v:0}, max_scn:{val:1723774797375521000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544217252}, prev_proposal_id:7, committed_end_lsn:{lsn:18544217374}, data_checksum:1913811101, accum_checksum:820719565, is_padding_log:false, is_raw_write:false}, state_map:{val:26}, ref_cnt:0, gen_ts:1723774797460412, freeze_ts:1723774797460412, submit_ts:1723774797460417, flushed_ts:1723774797461411, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:994}, ack_info_list=[]) [2024-08-16 10:19:57.557832] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=16] [PALF_DUMP](palf_id=1261, self="192.168.0.128:2882", [SlidingWindow]={palf_id:1261, self:"192.168.0.128:2882", lsn_allocator:{max_log_id:33675539, max_lsn:32601558870, max_scn:{val:1723774797358457001, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:32601558870}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:32559615830}}, last_submit_lsn:{lsn:32601558748}, last_submit_end_lsn:{lsn:32601558870}, last_submit_log_id:33675539, last_submit_log_pid:5, max_flushed_lsn:{lsn:32601558748}, max_flushed_end_lsn:{lsn:32601558870}, max_flushed_log_pid:5, committed_end_lsn:{lsn:32601558870}, last_slide_log_id:33675539, last_slide_scn:{val:1723774797358457001, v:0}, last_slide_lsn:{lsn:32601558748}, last_slide_end_lsn:{lsn:32601558870}, last_slide_log_pid:5, last_slide_log_accum_checksum:3712868123, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x7fa6fe292530}, [StateMgr]={this:0x7fa6fe2962b0, self:"192.168.0.128:2882", palf_id:1261, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:5}, leader:"192.168.0.125:2882", leader_epoch:9, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:32276646160}, scan_disk_log_finished:true, last_check_start_id:33675540, is_changing_config_with_arb:false, reconfirm_start_time_us:1722594182152738, palf_role_change_cb:0x7fa6fe29cb70, allow_vote:true}, [ConfigMgr]={palf_id:1261, self:"192.168.0.128:2882", alive_paxos_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, alive_paxos_replica_num:3, log_ms_meta:{version:3, proposal_id:5, prev:{version:1, config:{config_version:{proposal_id:4, config_seq:415}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29145, lock_time:1723617733782611}}, curr:{version:1, config:{config_version:{proposal_id:5, config_seq:416}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29145, lock_time:1723617733782611}}, prev_log_proposal_id:4, prev_lsn:{lsn:32276646038}, prev_mode_pid:5}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:4, prev_lsn:{lsn:32276415355}, prev_end_lsn:{lsn:32276415514}, prev_mode_pid:4}, reconfig_barrier:{prev_log_proposal_id:4, prev_lsn:{lsn:32276415355}, prev_end_lsn:{lsn:32276415514}, prev_mode_pid:4}, persistent_config_version:{proposal_id:5, config_seq:416}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1723774794248902, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1723774794248902, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:1722931663742194, this:0x7fa6fe293c70}, [ModeMgr]={palf_id:1261, self:"192.168.0.128:2882", applied_mode_meta:{version:1, proposal_id:5, mode_version:0, access_mode:1, ref_scn:{val:1720516541190358000, v:0}}, accepted_mode_meta:{version:1, proposal_id:5, mode_version:0, access_mode:1, ref_scn:{val:1720516541190358000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:5, mode_version:0, access_mode:1, ref_scn:{val:1720516541190358000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1261, is_inited:true, min_block_max_scn:{val:1721957059006967058, v:0}, min_block_id:421, min_block_min_scn:{val:1721956977387008021, v:0}, base_lsn_for_block_gc:{lsn:32545812480}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:5}, log_config_meta:{version:3, proposal_id:5, prev:{version:1, config:{config_version:{proposal_id:4, config_seq:415}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29145, lock_time:1723617733782611}}, curr:{version:1, config:{config_version:{proposal_id:5, config_seq:416}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29145, lock_time:1723617733782611}}, prev_log_proposal_id:4, prev_lsn:{lsn:32276646038}, prev_mode_pid:5}, log_snapshot_meta:{version:1, base_lsn:{lsn:32545812480}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:5, mode_version:0, access_mode:1, ref_scn:{val:1720516541190358000, v:0}}}, log_meta_storage:{log_tail:{lsn:1191936}, readable_log_tail:{lsn:1191936}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:32612917248}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1261}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1261/meta", dir_fd:199, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:65912832, block_header_serialize_buf_:0x7fa6fe298438, flashback_version:0}, log_storage:{log_tail:{lsn:32601558870}, readable_log_tail:{lsn:32601558870}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:32545812480}, min_scn:{val:1723732336322565001, v:0}, curr_block_id:485, palf_id:1261}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1261/log", dir_fd:200, min_block_id:421, max_block_id:486, curr_writable_block_id:485}, logical_block_size_:67104768, curr_block_writable_size_:11358378, block_header_serialize_buf_:0x7fa6fe29a0b8, flashback_version:0}, palf_epoch:3, last_purge_throttling_ts:1723452062503097, this:0x7fa6fe2969f0}, [Reconfirm]={palf_id:1261, self:"192.168.0.128:2882", state:"FINISHED", new_proposal_id:4, prepare_log_ack_list:2"192.168.0.125:2882";"192.168.0.133:2882";, curr_paxos_follower_list:2{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, majority_cnt:2, majority_max_log_server:"192.168.0.128:2882", majority_max_accept_pid:3, majority_max_lsn:{lsn:30467531174}, saved_end_lsn:{lsn:30467531174}, last_submit_prepare_req_time_us:1722594182153177, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x7fa6fe296430}) [2024-08-16 10:19:57.557980] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=143] [PALF_DUMP](palf_id=1261, self="192.168.0.128:2882", ack_info_list=[{member:{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}, {member:{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}, {member:{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}]) [2024-08-16 10:19:57.558004] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=22] current log_task status(palf_id=1261, self="192.168.0.128:2882", log_id=33675540, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[{member:{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}, {member:{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}, {member:{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}]) [2024-08-16 10:19:57.567272] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] ====== checkpoint timer task ====== [2024-08-16 10:19:57.568224] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=17] succ to get rec scn(*this={this:0x7fa707804030, ls_id:{id:1079}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1722510647991558001, v:0}, last_push_gc_task_ts_:1722511534898926, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:103}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:19:57.568282] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=55] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596c080, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1722511534901545, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596c080, snapshot_version:{val:1722511534901545, v:0}, ls_id:{id:1079}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:19:57.568326] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=35] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1722497541268834001, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1722497541268834001, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1079}) [2024-08-16 10:19:57.568393] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:265) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=27] [FREEZE]get rec_scn from MdsTableMgr(ret=0, ret="OB_SUCCESS", min_rec_scn={val:1723723445809886004, v:0}, order_flusher_for_one={high_priority_flusher:size:1, ordered_tablets:{{tablet_id:{id:208204}, rec_scn:{val:1723723445809886004, v:0}}, }, *this={this:0x7fa6fa246510, is_inited:true, freezing_scn:{val:1721997282211836001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774795649400002, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:19:57.568489] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=92] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="MDS_TABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={this:0x7fa6fa246510, is_inited:true, freezing_scn:{val:1721997282211836001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774795649400002, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}, min_rec_scn={val:1723723445809886004, v:0}, ls_id_={id:1079}) [2024-08-16 10:19:57.568559] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=47] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723723202639465000, v:0}, checkpoint_scn_in_ls_meta={val:1723723202639465000, v:0}, ls_id={id:1079}, service_type="DUP_TABLE") [2024-08-16 10:19:57.568588] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=14][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:18453811200}, new_base_lsn={lsn:18453811200}, lsn={lsn:18453811200}) [2024-08-16 10:19:57.568606] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=17] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:18453811200}, ls->get_ls_id()={id:1079}) [2024-08-16 10:19:57.569562] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] succ to get rec scn(*this={this:0x7fa7078b6030, ls_id:{id:1260}, tenant_id:1002, state:"L_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1723690101339443000, v:0}, last_push_gc_task_ts_:1723691902664029, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:6460}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:19:57.569587] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=24] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596eab0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1723691902666264, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596eab0, snapshot_version:{val:1723691902666264, v:0}, ls_id:{id:1260}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:19:57.569605] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=14] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1723690100831412000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1723690100831412000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1260}) [2024-08-16 10:19:57.571315] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:265) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] [FREEZE]get rec_scn from MdsTableMgr(ret=0, ret="OB_SUCCESS", min_rec_scn={val:1723723302639709002, v:0}, order_flusher_for_one={high_priority_flusher:size:1, ordered_tablets:{{tablet_id:{id:1152921504606868225}, rec_scn:{val:1723723302639709002, v:0}}, }, *this={this:0x7fa6bd53e510, is_inited:true, freezing_scn:{val:1723690100831412000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1260}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202608224000, v:0}, clog_base_lsn:{lsn:8723619840}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774795649400002, v:0}, tablet_change_checkpoint_scn:{val:1723719123293042000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723639313794159001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1260, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:19:57.571376] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=57] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x7fa6bd53e1b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x7fa6bd543cd0, table_type:1, memtable_head:149, memtable_tail:150, t3m:0x7fa7118b2030, tables:[null, null, null, null, null, 0x7fa70dc06cc0, null, null, null, null, null, null, null, null, null, null]}, is_freezing:false, ls_id:{id:1260}, mini_merge_recycle_commit_versions_ts:1723723202700999, tx_data_table:0x7fa6bd5451d0, ls_tablet_svr:0x7fa6bd53e190, slice_allocator:0x7fa6bd545210}, min_rec_scn={val:1723723202608224000, v:0}, ls_id_={id:1260}) [2024-08-16 10:19:57.576956] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=21] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723723202608224000, v:0}, checkpoint_scn_in_ls_meta={val:1723723202608224000, v:0}, ls_id={id:1260}, service_type="TRANS_SERVICE") [2024-08-16 10:19:57.576990] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:8723619840}, new_base_lsn={lsn:8723619840}, lsn={lsn:8723619840}) [2024-08-16 10:19:57.577001] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:8723619840}, ls->get_ls_id()={id:1260}) [2024-08-16 10:19:57.577942] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] succ to get rec scn(*this={this:0x7fa70da04030, ls_id:{id:1261}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1723712452247151000, v:0}, last_push_gc_task_ts_:1723712455858054, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:6477}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:19:57.577971] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=28] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596ce90, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1723723202513573, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596ce90, snapshot_version:{val:1723723202513573, v:0}, ls_id:{id:1261}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:19:57.577990] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=15] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1723654533322558000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1723654533322558000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1261}) [2024-08-16 10:19:57.578469] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=11] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x7fa6fe204510, is_inited:true, freezing_scn:{val:1723752041922970000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1261}, ls_create_status:1, clog_checkpoint_scn:{val:1723752188579788000, v:0}, clog_base_lsn:{lsn:32545812480}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774795649400002, v:0}, tablet_change_checkpoint_scn:{val:1723719128263216001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723617733349446001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:4, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1261}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1261, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:19:57.578541] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=68] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x7fa6fe2041b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x7fa6fe209cd0, table_type:1, memtable_head:173, memtable_tail:174, t3m:0x7fa7118b2030, tables:[null, null, null, null, null, null, null, null, null, null, null, null, null, 0x7fa70dc09900, null, null]}, is_freezing:false, ls_id:{id:1261}, mini_merge_recycle_commit_versions_ts:1723752042418277, tx_data_table:0x7fa6fe20b1d0, ls_tablet_svr:0x7fa6fe204190, slice_allocator:0x7fa6fe20b210}, min_rec_scn={val:1723752188579788000, v:0}, ls_id_={id:1261}) [2024-08-16 10:19:57.584019] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=29] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723752188579788000, v:0}, checkpoint_scn_in_ls_meta={val:1723752188579788000, v:0}, ls_id={id:1261}, service_type="TRANS_SERVICE") [2024-08-16 10:19:57.584047] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:32545812480}, new_base_lsn={lsn:32545812480}, lsn={lsn:32545812480}) [2024-08-16 10:19:57.584057] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:32545812480}, ls->get_ls_id()={id:1261}) [2024-08-16 10:19:57.584199] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=13] ====== check clog disk timer task ====== [2024-08-16 10:19:57.584219] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=17] get_disk_usage(ret=0, capacity(MB):=2150, used(MB):=0) [2024-08-16 10:19:57.584234] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=9] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=676457349, need_update_checkpoint_scn=false) [2024-08-16 10:19:57.584917] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] succ to get rec scn(*this={this:0x7fa70dab6030, ls_id:{id:1262}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1723712432019955000, v:0}, last_push_gc_task_ts_:1723712435838837, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:6328}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:19:57.584942] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=24] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596dca0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1723712435841010, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596dca0, snapshot_version:{val:1723712435841010, v:0}, ls_id:{id:1262}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:19:57.584960] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=14] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1723657339856816000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1723657339856816000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1262}) [2024-08-16 10:19:57.585341] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=6] ====== check clog disk timer task ====== [2024-08-16 10:19:57.585361] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=18] get_disk_usage(ret=0, capacity(MB):=0, used(MB):=0) [2024-08-16 10:19:57.585375] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=9] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=0, need_update_checkpoint_scn=false) [2024-08-16 10:19:57.586557] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:265) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=13] [FREEZE]get rec_scn from MdsTableMgr(ret=0, ret="OB_SUCCESS", min_rec_scn={val:1723723207973113002, v:0}, order_flusher_for_one={high_priority_flusher:size:1, ordered_tablets:{{tablet_id:{id:209134}, rec_scn:{val:1723723207973113002, v:0}}, }, *this={this:0x7fa6be4fc510, is_inited:true, freezing_scn:{val:1723656976361279000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774795649400002, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:19:57.586622] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=62] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="MDS_TABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={this:0x7fa6be4fc510, is_inited:true, freezing_scn:{val:1723656976361279000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774795649400002, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}, min_rec_scn={val:1723723207973113002, v:0}, ls_id_={id:1262}) [2024-08-16 10:19:57.590887] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=15][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:19:57.591951] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=38] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723723207973113002, v:0}, checkpoint_scn_in_ls_meta={val:1723723207973113002, v:0}, ls_id={id:1262}, service_type="TRANS_SERVICE") [2024-08-16 10:19:57.591988] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=27][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:8253886464}, new_base_lsn={lsn:8253886464}, lsn={lsn:8253886464}) [2024-08-16 10:19:57.591999] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:8253886464}, ls->get_ls_id()={id:1262}) [2024-08-16 10:19:57.592006] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:191) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] succeed to update_clog_checkpoint(ret=0, ls_cnt=4) [2024-08-16 10:19:57.593216] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=24] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:19:57.593319] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=34][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:19:57.593341] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=21][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:19:57.593349] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=7] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:19:57.597721] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.597749] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=26] handle task succ(id={id:1260}) [2024-08-16 10:19:57.597760] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.597768] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.597783] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.606736] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=20] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:19:57.614322] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [25973][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=27] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:19:57.616677] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=26] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=1, size_used=60668, mem_used=137263104) [2024-08-16 10:19:57.627775] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=30] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=5) [2024-08-16 10:19:57.641497] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:57.641602] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:57.641618] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:57.641625] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:57.642835] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE587-0-0] [lt=23] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=6, avg_log_gen_to_submit_time=67, avg_log_submit_to_flush_time=705, avg_log_submit_to_slide_time=1169) [2024-08-16 10:19:57.642873] INFO [LIB] stat (utility.h:1197) [21388][T1002_ApplySrv3][T1002][YB42C0A80080-00061CCCC52E0943-0-0] [lt=3] [APPLY STAT CB APPEND COST TIME](cur_stat_count=49, stat_interval=5000000, avg (count/cost)=112, this=0x7fa70bda0280, extra_info="") [2024-08-16 10:19:57.642896] INFO [LIB] stat (utility.h:1197) [21388][T1002_ApplySrv3][T1002][YB42C0A80080-00061CCCC52E0943-0-0] [lt=22] [APPLY STAT CB IN QUEUE TIME](cur_stat_count=49, stat_interval=5000000, avg (count/cost)=1225, this=0x7fa70bda04b0, extra_info="") [2024-08-16 10:19:57.642901] INFO [LIB] stat (utility.h:1197) [21388][T1002_ApplySrv3][T1002][YB42C0A80080-00061CCCC52E0943-0-0] [lt=5] [APPLY STAT CB WAIT COMMIT TIME](cur_stat_count=49, stat_interval=5000000, avg (count/cost)=0, this=0x7fa70bda06e0, extra_info="") [2024-08-16 10:19:57.642908] INFO [LIB] stat (utility.h:1197) [21388][T1002_ApplySrv3][T1002][YB42C0A80080-00061CCCC52E0943-0-0] [lt=7] [APPLY STAT CB EXECUTE TIME](cur_stat_count=49, stat_interval=5000000, avg (count/cost)=-533, this=0x7fa70bda0910, extra_info="") [2024-08-16 10:19:57.642925] INFO [LIB] stat (utility.h:1197) [21388][T1002_ApplySrv3][T1002][YB42C0A80080-00061CCCC52E0943-0-0] [lt=15] [APPLY STAT CB TOTAL TIME](cur_stat_count=49, stat_interval=5000000, avg (count/cost)=804, this=0x7fa70bda0b40, extra_info="") [2024-08-16 10:19:57.643176] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:57.649316] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.649335] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:57.649345] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.649355] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.649374] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.651729] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74346-0-0] [lt=22] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:19:57.651748] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74346-0-0] [lt=18] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1071, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:19:57.660585] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFE1-0-0] [lt=7][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509826, log_task={header:{begin_lsn:{lsn:18544217374}, end_lsn:{lsn:18544217496}, log_id:53509826, min_scn:{val:1723774797375521000, v:0}, max_scn:{val:1723774797375521000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544217252}, prev_proposal_id:7, committed_end_lsn:{lsn:18544217374}, data_checksum:1913811101, accum_checksum:820719565, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774797460412, freeze_ts:1723774797460412, submit_ts:1723774797460417, flushed_ts:1723774797461411, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:994}, fs_cb_begin_ts=1723774797660572, log_life_time=200160) [2024-08-16 10:19:57.661199] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1623) [2024-08-16 10:19:57.697797] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.697825] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:19:57.697841] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.697855] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.697875] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.733132] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:57.733388] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash time detail, (compute_wash_size_time=152, refresh_score_time=68, wash_time=16) [2024-08-16 10:19:57.736580] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=817882, clean_num=62914) [2024-08-16 10:19:57.741564] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] try gc retain ctx [2024-08-16 10:19:57.741599] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:57.741700] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=43, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:19:57.741736] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:19:57.741750] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:57.741759] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:57.741765] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:57.743279] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=24][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:57.749387] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.749407] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:57.749418] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.749427] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.749440] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.760699] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFE7-0-0] [lt=44] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:19:57.760760] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFE7-0-0] [lt=29] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:19:57.760775] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFE7-0-0] [lt=15] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=100113, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:19:57.760787] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFE7-0-0] [lt=10][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509827, log_task={header:{begin_lsn:{lsn:18544217496}, end_lsn:{lsn:18544217618}, log_id:53509827, min_scn:{val:1723774797474131000, v:0}, max_scn:{val:1723774797474131000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544217374}, prev_proposal_id:7, committed_end_lsn:{lsn:18544217496}, data_checksum:1913811101, accum_checksum:3034808062, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774797660551, freeze_ts:1723774797660551, submit_ts:1723774797660557, flushed_ts:1723774797661262, gen_to_freeze cost time:0, gen_to_submit cost time:6, submit_to_flush cost time:705}, fs_cb_begin_ts=1723774797760744, log_life_time=100193) [2024-08-16 10:19:57.763486] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=37] ====== tenant freeze timer task ====== [2024-08-16 10:19:57.763517] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=19][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:19:57.797863] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.797910] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=44] handle task succ(id={id:1260}) [2024-08-16 10:19:57.797920] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.797929] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.797944] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.827882] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=21] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=4) [2024-08-16 10:19:57.841641] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:57.841739] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:57.841754] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:57.841762] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:57.842339] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:19:57.843348] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:57.849463] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.849484] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:57.849494] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.849503] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.849516] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.858353] INFO [SERVER] try_reload_schema (ob_server_schema_updater.cpp:440) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=15] schedule fetch new schema task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:19:57.858378] INFO [SERVER] do_heartbeat_event (ob_heartbeat.cpp:268) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=24] try reload schema success(schema_version=1722933080671200, refresh_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_ret=0) [2024-08-16 10:19:57.858410] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:255) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E7C-0-0] [lt=9] [REFRESH_SCHEMA] start to process schema refresh task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:19:57.858429] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:280) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E7C-0-0] [lt=15] [REFRESH_SCHEMA] local schema info is newer, no need to refresh schema(ret=0, ret="OB_SUCCESS", local_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:19:57.861007] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFED-0-0] [lt=42][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509828, log_task={header:{begin_lsn:{lsn:18544217618}, end_lsn:{lsn:18544217740}, log_id:53509828, min_scn:{val:1723774797661016000, v:0}, max_scn:{val:1723774797661016000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544217496}, prev_proposal_id:7, committed_end_lsn:{lsn:18544217618}, data_checksum:1913811101, accum_checksum:2262775645, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774797760685, freeze_ts:1723774797760685, submit_ts:1723774797760730, flushed_ts:1723774797761365, gen_to_freeze cost time:0, gen_to_submit cost time:45, submit_to_flush cost time:635}, fs_cb_begin_ts=1723774797860996, log_life_time=100311) [2024-08-16 10:19:57.861620] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=35] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=636, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:19:57.861872] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2752][T1001_TenantMet][T1001][Y0-0000000000000000-0-0] [lt=41] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=0, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:19:57.865811] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [31770][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=42] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1, server_version={version:{val:1723774792918301000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0}, server_version_delta=4947507, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.126:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29977620, succ_cluster_heartbeat_count=29977602, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774342455102000, v:0}, local_cluster_delta=455410706, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:19:57.875092] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [2536][T1002_LockWaitM][T1002][Y0-0000000000000000-0-0] [lt=21] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-08-16 10:19:57.875203] INFO [PALF] read (log_cache.cpp:100) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=34] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:57.897949] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.897974] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:19:57.897985] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.897997] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.898011] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.901987] INFO [CLOG] do_thread_task_ (ob_remote_fetch_log_worker.cpp:250) [2762][T1001_RFLWorker][T1001][YB42C0A80080-00061CCC92E1D0C5-0-0] [lt=30] ObRemoteFetchWorker is running(thread_index=0) [2024-08-16 10:19:57.908801] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [31504][DumpLSLoc][T0][YB42C0A80080-00061CCC8F16CA28-0-0] [lt=253] [LS_LOCATION]dump tenant ls location caches(tenant_id=1, tenant_ls_locations=[{cache_key:{tenant_id:1, ls_id:{id:1}, cluster_id:1713182723}, renew_time:1723774797528332, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:11}, {server:"192.168.0.129:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.134:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}]) [2024-08-16 10:19:57.908862] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [31504][DumpLSLoc][T0][YB42C0A80080-00061CCC8F16CA28-0-0] [lt=56] [LS_LOCATION]dump tenant ls location caches(tenant_id=1001, tenant_ls_locations=[{cache_key:{tenant_id:1001, ls_id:{id:1}, cluster_id:1713182723}, renew_time:1723774797530556, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:7}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}]) [2024-08-16 10:19:57.908948] INFO [SHARE] blacklist_loop_ (ob_server_blacklist.cpp:313) [31526][Blacklist][T0][Y0-0000000000000000-0-0] [lt=28] blacklist_loop exec finished(cost_time=19, is_enabled=true, send_cnt=0) [2024-08-16 10:19:57.908965] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [31504][DumpLSLoc][T0][YB42C0A80080-00061CCC8F16CA28-0-0] [lt=35] [LS_LOCATION]dump tenant ls location caches(tenant_id=1002, tenant_ls_locations=[{cache_key:{tenant_id:1002, ls_id:{id:1312}, cluster_id:1713182723}, renew_time:1723105980208625, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1294}, cluster_id:1713182723}, renew_time:1722595652102251, replica_locations:[{server:"192.168.0.124:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1363}, cluster_id:1713182723}, renew_time:1723646553055685, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1292}, cluster_id:1713182723}, renew_time:1722594499160155, replica_locations:[{server:"192.168.0.124:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1286}, cluster_id:1713182723}, renew_time:1722594010522003, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1309}, cluster_id:1713182723}, renew_time:1722832077430884, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1300}, cluster_id:1713182723}, renew_time:1722595777348102, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1325}, cluster_id:1713182723}, renew_time:1723452113048028, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1350}, cluster_id:1713182723}, renew_time:1723617735379749, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1316}, cluster_id:1713182723}, renew_time:1723451932706270, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1280}, cluster_id:1713182723}, renew_time:1722593548703641, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:7}]}, {cache_key:{tenant_id:1002, ls_id:{id:1291}, cluster_id:1713182723}, renew_time:1722594313540279, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1290}, cluster_id:1713182723}, renew_time:1722594318555608, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.134:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1339}, cluster_id:1713182723}, renew_time:1723516963339645, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1284}, cluster_id:1713182723}, renew_time:1722593985405302, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.134:2882", role:2, sql_port:2881, replica_type:16, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1344}, cluster_id:1713182723}, renew_time:1723517113628829, replica_locations:[{server:"192.168.0.125:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1359}, cluster_id:1713182723}, renew_time:1723618040940888, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1330}, cluster_id:1713182723}, renew_time:1723516813046198, replica_locations:[{server:"192.168.0.124:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1261}, cluster_id:1713182723}, renew_time:1723774797533198, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:5}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1028}, cluster_id:1713182723}, renew_time:1723774797533184, replica_locations:[{server:"192.168.0.124:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:6}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1361}, cluster_id:1713182723}, renew_time:1723639351369428, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1271}, cluster_id:1713182723}, renew_time:1720533509017285, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1305}, cluster_id:1713182723}, renew_time:1722595958684187, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1269}, cluster_id:1713182723}, renew_time:1720533468951712, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1348}, cluster_id:1713182723}, renew_time:1723617720354819, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1337}, cluster_id:1713182723}, renew_time:1723516903210675, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1298}, cluster_id:1713182723}, renew_time:1722595737258877, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1329}, cluster_id:1713182723}, renew_time:1723516803029599, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1314}, cluster_id:1713182723}, renew_time:1723451932706267, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:3}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1268}, cluster_id:1713182723}, renew_time:1720533488989721, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1274}, cluster_id:1713182723}, renew_time:1720591144595730, replica_locations:[{server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1331}, cluster_id:1713182723}, renew_time:1723516803029649, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}]) [2024-08-16 10:19:57.918899] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=23] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:19:57.931484] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80081-00061EFF86EC9419-0-0] [lt=23][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:57.937036] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:57.937335] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=129, refresh_score_time=102, wash_time=17) [2024-08-16 10:19:57.940470] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=880796, clean_num=62914) [2024-08-16 10:19:57.941723] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:57.941841] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:19:57.941875] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1220) [2024-08-16 10:19:57.941894] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:57.941902] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:57.941907] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:57.943433] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:57.949542] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.949563] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:57.949574] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.949584] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.949599] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:57.950888] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=4] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:57.950926] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=26] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:57.951102] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:19:57.951126] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=17][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:19:57.951136] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:19:57.951144] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=7][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:19:57.951998] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7434F-0-0] [lt=41] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=617, avg_log_submit_to_slide_time=1053) [2024-08-16 10:19:57.952030] INFO [PALF] read (log_cache.cpp:100) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=57] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:57.961396] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=19] ====== check clog disk timer task ====== [2024-08-16 10:19:57.961429] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=25] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:57.961452] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=10] cannot_recycle_log_size statistics(cannot_recycle_log_size=391624560, threshold=6088116142, need_update_checkpoint_scn=false) [2024-08-16 10:19:57.964162] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [31728][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=79] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:19:57.975310] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE592-0-0] [lt=21] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=680, avg_log_submit_to_slide_time=1072) [2024-08-16 10:19:57.998022] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:57.998042] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:57.998053] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:57.998062] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:57.998092] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.012055] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=42] gc stale ls task succ [2024-08-16 10:19:58.012171] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=5] gc stale ls task succ [2024-08-16 10:19:58.012200] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=19] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:19:58.012217] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=10] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:19:58.012228] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=9] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:19:58.016126] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:58.016630] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:58.023514] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C06-0-0] [lt=21] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774798023458) [2024-08-16 10:19:58.027986] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=24] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=3) [2024-08-16 10:19:58.041677] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=36] PNIO [ratelimit] time: 1723774798041675, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007618, add_bytes: 0 [2024-08-16 10:19:58.041800] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:58.041914] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:58.041930] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:58.041938] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:58.043513] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=24][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:58.044479] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:19:58.049374] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2417][T1002_Occam][T1002][YB42C0A80080-00061CCCBF7D385E-0-0] [lt=22] for each ls(succ_num=4, ret=0, ret="OB_SUCCESS") [2024-08-16 10:19:58.049619] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.049641] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:58.049650] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.049667] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.049682] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.050488] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=24] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:19:58.051702] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:19:58.052224] INFO [CLOG] handle_submit_task_ (ob_log_replay_service.cpp:1208) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=12] succ to submit log task to replay service(to_submit_lsn={lsn:8394841226}, to_submit_scn={val:1723774797753355002, v:0}, replay_status={ls_id_:{id:1262}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774798052196, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10679383, ret_code_:0}}, next_to_submit_lsn_:{lsn:8394841292}, next_to_submit_scn_:{val:1723774797753355003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516551920929002, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69ec05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8394841170}, end_lsn:{lsn:8394841292}, read_buf:{buf_len_:3682304, buf_:0x7fa69ec05000}, block_size:67104768, log_storage_:0x7fa6be533470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774797753355002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774797753355002, v:0}, data_checksum:2892311396, flag:0}}, init_mode_version:0, accumulate_checksum:1615663449, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774797753355002, v:0}}}}}) [2024-08-16 10:19:58.061041] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCEFF9-0-0] [lt=26][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509830, log_task={header:{begin_lsn:{lsn:18544217862}, end_lsn:{lsn:18544217984}, log_id:53509830, min_scn:{val:1723774797875828000, v:0}, max_scn:{val:1723774797875828000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544217740}, prev_proposal_id:7, committed_end_lsn:{lsn:18544217862}, data_checksum:1913811101, accum_checksum:1810414498, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774797960916, freeze_ts:1723774797960916, submit_ts:1723774797960920, flushed_ts:1723774797962038, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:1118}, fs_cb_begin_ts=1723774798061025, log_life_time=100109) [2024-08-16 10:19:58.061135] INFO [LIB] stat (utility.h:1197) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=31] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=29, stat_interval=1000000, avg (count/cost)=43, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:19:58.061156] INFO [LIB] stat (utility.h:1197) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=20] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=29, stat_interval=1000000, avg (count/cost)=5, this=0x7fa713668b28, extra_info="") [2024-08-16 10:19:58.061725] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=224, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:19:58.064728] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=31][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:19:58.064755] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=25][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:19:58.074293] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=22] PNIO [ratelimit] time: 1723774798074291, bytes: 1496436055309, bw: 0.148249 MB/s, add_ts: 1001028, add_bytes: 155610 [2024-08-16 10:19:58.074338] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85C0-0-0] [lt=73] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:19:58.096108] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [2785][T1001_ObTimer][T1001][Y0-0000000000000000-0-0] [lt=8] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:19:58.096139] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:230) [2785][T1001_ObTimer][T1001][Y0-0000000000000000-0-0] [lt=23] [tabletchange] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12274) [2024-08-16 10:19:58.098111] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.098132] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:19:58.098151] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:19:58.098165] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.098179] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.098209] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.115987] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AAB-0-0] [lt=56] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:19:58.116027] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AAB-0-0] [lt=25] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774797753355002, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774797753355002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:19:58.120743] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7D8-0-0] [lt=65] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:19:58.123865] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2659][T1001_Occam][T1001][YB42C0A80080-00061CCCBF8D385E-0-0] [lt=22] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:19:58.136641] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAD6-0-0] [lt=31][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=704, pos=350, bitmap=NULL) [2024-08-16 10:19:58.140911] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:58.141181] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=128, refresh_score_time=64, wash_time=15) [2024-08-16 10:19:58.141880] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:58.142012] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:58.142028] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:58.142036] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:58.143589] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:120) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22] tenant info is invalid(ret=-4076, query_ts=0, standby_scn={val:0, v:0}, this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:19:58.143621] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=30][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:58.144519] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=943710, clean_num=62914) [2024-08-16 10:19:58.149701] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.149728] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:19:58.149739] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.149749] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.149763] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.151181] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A89-0-0] [lt=29] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:19:58.151208] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A89-0-0] [lt=25] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:19:58.157926] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=20] ====== tenant freeze timer task ====== [2024-08-16 10:19:58.157978] INFO [STORAGE] get_ls_tx_data_memory_info_ (ob_tenant_freezer.cpp:665) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=17] TxData Memory Statistic(logstream info): (ls_id={id:1079}, Frozen TxData Memory(MB)=0, Active TxData Memory(MB)=2) [2024-08-16 10:19:58.157998] INFO [STORAGE] get_ls_tx_data_memory_info_ (ob_tenant_freezer.cpp:665) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=17] TxData Memory Statistic(logstream info): (ls_id={id:1260}, Frozen TxData Memory(MB)=0, Active TxData Memory(MB)=5) [2024-08-16 10:19:58.158006] INFO [STORAGE] get_ls_tx_data_memory_info_ (ob_tenant_freezer.cpp:665) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=7] TxData Memory Statistic(logstream info): (ls_id={id:1261}, Frozen TxData Memory(MB)=0, Active TxData Memory(MB)=1) [2024-08-16 10:19:58.158011] INFO [STORAGE] get_ls_tx_data_memory_info_ (ob_tenant_freezer.cpp:665) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=4] TxData Memory Statistic(logstream info): (ls_id={id:1262}, Frozen TxData Memory(MB)=0, Active TxData Memory(MB)=4) [2024-08-16 10:19:58.158016] INFO [STORAGE] check_and_freeze_tx_data_ (ob_tenant_freezer.cpp:571) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=3] TxData Memory Statistic : (Tenant Total Memory(MB)=6144, Tenant Frozen TxData Memory(MB)=0, Tenant Active TxData Memory(MB)=13, Freeze TxData Trigger Memory(MB)=122, Total TxDataTable Hold Memory(MB)=60, Total TxDataTable Memory Limit(MB)=614) [2024-08-16 10:19:58.159681] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007F-000616D421E91A0C-0-0] [lt=22][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:58.171606] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAD7-0-0] [lt=25][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=1, len=49, pos=18, bitmap=NULL) [2024-08-16 10:19:58.174407] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85C2-0-0] [lt=10] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:19:58.174611] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [2549][T1002_LCLSender][T1002][Y0-0000000000000000-0-0] [lt=27] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=1135, total_destructed_detector=1135, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x7fa7110382b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-08-16 10:19:58.183293] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=26] gc stale ls task succ [2024-08-16 10:19:58.198187] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=29] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.198213] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:19:58.198223] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.198233] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.198248] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.213243] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [2786][T1001_TabletShe][T1001][Y0-0000000000000000-0-0] [lt=9] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:19:58.213273] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:101) [2786][T1001_TabletShe][T1001][Y0-0000000000000000-0-0] [lt=23] [emptytablet] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12274) [2024-08-16 10:19:58.228100] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=19] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=2) [2024-08-16 10:19:58.234049] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE7D-0-0] [lt=18] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:19:58.234105] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE7D-0-0] [lt=38] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774797858677003, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774797858677003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:19:58.236237] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=24] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:58.236240] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:58.236378] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=15] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:58.236395] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=17] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:58.236400] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:58.236772] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=19] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:58.237062] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=18] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:58.241950] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:58.242061] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:58.242103] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=39] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:58.242113] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:58.242706] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=800, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:19:58.242738] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=48, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:19:58.242771] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=14, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:19:58.243246] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73567-0-0] [lt=22] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:19:58.243265] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73567-0-0] [lt=18] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1383, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:19:58.243690] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:58.249739] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.249760] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:58.249769] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.249778] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.249799] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.258090] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=30] re_register_parent reason(palf_id=1262, self="192.168.0.128:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1723774794248824, last_first_register_time_us=1720517744275280, register_time_us=1723774794248824, curr_time_us=1723774798258087) [2024-08-16 10:19:58.258161] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=34] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1262, self="192.168.0.128:2882", reason=1, leader="192.168.0.133:2882", register_time_us=1723774798258127, last_submit_register_req_time_us=1723774798258127, register_parent_reason=1) [2024-08-16 10:19:58.258197] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=33] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1262, self="192.168.0.128:2882") [2024-08-16 10:19:58.258217] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=8] re_register_parent reason(palf_id=1261, self="192.168.0.128:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1723774794248902, last_first_register_time_us=1720517744277428, register_time_us=1723774794248902, curr_time_us=1723774798258216) [2024-08-16 10:19:58.258254] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=12] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1261, self="192.168.0.128:2882", reason=1, leader="192.168.0.125:2882", register_time_us=1723774798258230, last_submit_register_req_time_us=1723774798258230, register_parent_reason=1) [2024-08-16 10:19:58.258269] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=15] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1261, self="192.168.0.128:2882") [2024-08-16 10:19:58.261055] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF005-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1501, avg_group_log_size=150) [2024-08-16 10:19:58.261143] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF005-0-0] [lt=72][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509833, log_task={header:{begin_lsn:{lsn:18544218509}, end_lsn:{lsn:18544218631}, log_id:53509833, min_scn:{val:1723774797875828003, v:0}, max_scn:{val:1723774797875828003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544218106}, prev_proposal_id:7, committed_end_lsn:{lsn:18544218509}, data_checksum:1913811101, accum_checksum:2147391342, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774798161042, freeze_ts:1723774798161042, submit_ts:1723774798161048, flushed_ts:1723774798161667, gen_to_freeze cost time:0, gen_to_submit cost time:6, submit_to_flush cost time:619}, fs_cb_begin_ts=1723774798261133, log_life_time=100091) [2024-08-16 10:19:58.261567] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=24] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574616, offset=42141401}, end_cursor:ObLogCursor{file_id=65, log_id=18574617, offset=42141642}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751684050, len:2343, log_data_len:241, seq:18574616, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:58.263191] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=36] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574617, offset=42141642}, end_cursor:ObLogCursor{file_id=65, log_id=18574618, offset=42141891}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75161c050, len:2102, log_data_len:249, seq:18574617, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:58.264826] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA0-0-0] [lt=19] push gts task success(*task={:""}) [2024-08-16 10:19:58.270507] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=27] current monitor number(seq_=-1) [2024-08-16 10:19:58.272299] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA0-0-0] [lt=22] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:19:58.273880] INFO [STORAGE.TRANS] process (ob_trans_rpc.cpp:142) [2809][T1001_L0_G0][T1001][YB42C0A8007C-000616D3F7C3A832-0-0] [lt=24] transaction rpc statistics(total_rt=236, total_process=10, avg_rt=21) [2024-08-16 10:19:58.274560] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85C4-0-0] [lt=32] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:19:58.274630] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=34] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574618, offset=42141891}, end_cursor:ObLogCursor{file_id=65, log_id=18574619, offset=42142140}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516cc050, len:1853, log_data_len:249, seq:18574618, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:58.275088] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=591, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:19:58.275117] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:19:58.275146] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=14] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=26, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:19:58.275202] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=17] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=38, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:19:58.275393] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=25] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574619, offset=42142140}, end_cursor:ObLogCursor{file_id=65, log_id=18574620, offset=42142389}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516e8050, len:1604, log_data_len:249, seq:18574619, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:58.275604] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7435B-0-0] [lt=18] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=14, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:19:58.275620] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7435B-0-0] [lt=15] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1059, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:19:58.298271] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.298298] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:19:58.298311] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.298321] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.298335] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.299352] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=15] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:19:58.299382] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=27] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=4996, mem_used=39515136) [2024-08-16 10:19:58.302855] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=19] ====== tenant freeze timer task ====== [2024-08-16 10:19:58.302891] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=21][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:19:58.317278] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=17] get all snapshots(ret=0, snapshots.count()=192, snapshots=[{snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213191, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213192, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213193, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213194, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213195, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213196, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213197, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213198, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213199, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213200, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213201, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213202, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213203, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213204, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213205, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213206, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213207, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213208, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213209, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213210, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213211, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213212, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213213, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213214, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213215, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213216, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213217, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213218, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213219, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213220, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213221, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213222, comment:null}]) [2024-08-16 10:19:58.342021] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:58.342148] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:58.342165] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:58.342173] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:58.343771] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=34][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:58.345008] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=24] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:58.345301] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=117, refresh_score_time=89, wash_time=26) [2024-08-16 10:19:58.348633] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1006624, clean_num=62914) [2024-08-16 10:19:58.349809] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.349830] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:58.349839] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.349851] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.349894] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.361207] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF00B-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1501) [2024-08-16 10:19:58.361243] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF00B-0-0] [lt=21] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=9, avg_log_submit_to_flush_time=859, avg_log_submit_to_slide_time=100095) [2024-08-16 10:19:58.361271] INFO [PALF] read (log_cache.cpp:100) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1501, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:58.361262] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF00B-0-0] [lt=17][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509834, log_task={header:{begin_lsn:{lsn:18544218631}, end_lsn:{lsn:18544218753}, log_id:53509834, min_scn:{val:1723774797875828004, v:0}, max_scn:{val:1723774797875828004, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544218509}, prev_proposal_id:7, committed_end_lsn:{lsn:18544218631}, data_checksum:1913811101, accum_checksum:2339144701, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774798261035, freeze_ts:1723774798261035, submit_ts:1723774798261041, flushed_ts:1723774798262933, gen_to_freeze cost time:0, gen_to_submit cost time:6, submit_to_flush cost time:1892}, fs_cb_begin_ts=1723774798361232, log_life_time=100197) [2024-08-16 10:19:58.383791] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=32] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:19:58.398365] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.398394] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=26] handle task succ(id={id:1260}) [2024-08-16 10:19:58.398407] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.398420] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.398448] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.419435] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=26] gts statistics(tenant_id=1, gts_rpc_cnt=15, get_gts_cache_cnt=102, get_gts_with_stc_cnt=5, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-08-16 10:19:58.419479] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=42] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1, need_refresh=false, gts_local_cache={srr:[mts=1723774797918850], gts:1723774797918304000, latest_srr:[mts=1723774798419403]}) [2024-08-16 10:19:58.419521] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=22] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_refresh=false, gts_local_cache={srr:[mts=1723774798266651], gts:1723774798264980000, latest_srr:[mts=1723774798419503]}) [2024-08-16 10:19:58.419741] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=9] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_refresh=false, gts_local_cache={srr:[mts=1723774797919129], gts:1723774797917543000, latest_srr:[mts=1723774798419717]}) [2024-08-16 10:19:58.424964] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=24] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:19:58.425065] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=18] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:19:58.427436] INFO [SHARE.LOCATION] renew_all_ls_locations_by_rpc (ob_ls_location_service.cpp:783) [31503][AutoLSLocRpc][T0][YB42C0A80080-00061CCCC718F0B6-0-0] [lt=114] [LS_LOCATION] Get ls leaders by RPC(ret=0, ret="OB_SUCCESS", dests=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882", "192.168.0.124:2882", "192.168.0.125:2882", "192.168.0.127:2882", "192.168.0.128:2882", "192.168.0.131:2882", "192.168.0.133:2882"], leaders=[{key:{tenant_id:1, ls_id:{id:1}, cluster_id:1713182723}, location:{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:11}}, {key:{tenant_id:1001, ls_id:{id:1}, cluster_id:1713182723}, location:{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:7}}, {key:{tenant_id:1002, ls_id:{id:1024}, cluster_id:1713182723}, location:{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:47}}, {key:{tenant_id:1002, ls_id:{id:1}, cluster_id:1713182723}, location:{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:7}}, {key:{tenant_id:1002, ls_id:{id:1079}, cluster_id:1713182723}, location:{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:7}}, {key:{tenant_id:1002, ls_id:{id:1261}, cluster_id:1713182723}, location:{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:5}}, {key:{tenant_id:1002, ls_id:{id:1028}, cluster_id:1713182723}, location:{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:6}}, {key:{tenant_id:1002, ls_id:{id:1260}, cluster_id:1713182723}, location:{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:29}}, {key:{tenant_id:1002, ls_id:{id:1026}, cluster_id:1713182723}, location:{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:20}}, {key:{tenant_id:1002, ls_id:{id:1262}, cluster_id:1713182723}, location:{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}}]) [2024-08-16 10:19:58.428205] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=23] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=1) [2024-08-16 10:19:58.442091] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:58.442185] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:58.442199] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:58.442213] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:58.443846] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:58.449887] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.449914] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:19:58.449923] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.449932] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.449946] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.461474] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF011-0-0] [lt=38][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509835, log_task={header:{begin_lsn:{lsn:18544218753}, end_lsn:{lsn:18544218875}, log_id:53509835, min_scn:{val:1723774797875828005, v:0}, max_scn:{val:1723774797875828005, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544218631}, prev_proposal_id:7, committed_end_lsn:{lsn:18544218753}, data_checksum:1913811101, accum_checksum:4082128606, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774798361189, freeze_ts:1723774798361189, submit_ts:1723774798361195, flushed_ts:1723774798361747, gen_to_freeze cost time:0, gen_to_submit cost time:6, submit_to_flush cost time:552}, fs_cb_begin_ts=1723774798461463, log_life_time=100274) [2024-08-16 10:19:58.498447] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.498474] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:19:58.498484] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.498495] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.498509] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.516034] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [31763][T1_LCLSender][T1][Y0-0000000000000000-0-0] [lt=27] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=1135, total_destructed_detector=1135, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x7fa76bdfe2b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-08-16 10:19:58.525266] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=27] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=6888, mem_used=12478464) [2024-08-16 10:19:58.532704] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=22] start do ls ha handler(ls_id_array_=[{id:1260}, {id:1262}, {id:1079}, {id:1261}]) [2024-08-16 10:19:58.542136] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:58.542256] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:58.542305] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=46] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:58.542314] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:58.543679] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [31738][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=45] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:940, allocator used:3737440, allocator total:3859072}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:140, allocator used:9182740, allocator total:9182740}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=13041812, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:63, allocator used:124992, allocator total:130816}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:19:58.543920] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:58.548142] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 346.00) [2024-08-16 10:19:58.548167] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=19] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:58.548250] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=33] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.98, rt: 839.75) [2024-08-16 10:19:58.548278] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=22] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:58.548317] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=31] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548342] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=22] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548352] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548368] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=14] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548377] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548384] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548391] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548408] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548418] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548434] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548444] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548475] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=29] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548495] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=18] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548504] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548521] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.548531] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:58.549097] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:58.549374] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=34] Wash time detail, (compute_wash_size_time=159, refresh_score_time=80, wash_time=16) [2024-08-16 10:19:58.549924] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.549947] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:19:58.549958] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.549977] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.549998] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.551596] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A91-0-0] [lt=77] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:19:58.552199] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=532, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:19:58.552249] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=31] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=31, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:19:58.552295] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=34, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:19:58.552848] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1069538, clean_num=62914) [2024-08-16 10:19:58.590962] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=14][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:19:58.593237] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=29] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:19:58.593367] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=22][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:19:58.593392] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=23][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:19:58.593401] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=8] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:19:58.598521] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=32] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.598540] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] handle task succ(id={id:1260}) [2024-08-16 10:19:58.598549] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.598558] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.598571] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.606854] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=14] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:19:58.616800] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=29] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60672, mem_used=137263104) [2024-08-16 10:19:58.617015] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=28] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:19:58.628310] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=26] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=0) [2024-08-16 10:19:58.642185] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:58.642288] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:58.642310] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:58.642317] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:58.643611] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73573-0-0] [lt=20] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=5, avg_log_gen_to_submit_time=62, avg_log_submit_to_flush_time=871, avg_log_submit_to_slide_time=1336) [2024-08-16 10:19:58.644009] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:58.649995] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.650016] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:58.650026] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.650036] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.650050] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.652560] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74365-0-0] [lt=29] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:19:58.652613] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74365-0-0] [lt=52] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1010, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:19:58.698614] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.698641] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:19:58.698653] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.698665] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.698682] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.742260] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:58.742383] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=31] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=44, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:19:58.742420] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:19:58.742433] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:58.742442] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:58.742454] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:58.744087] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:58.750056] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.750085] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:19:58.750095] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.750104] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.750117] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.753357] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:58.753681] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=40] Wash time detail, (compute_wash_size_time=130, refresh_score_time=97, wash_time=19) [2024-08-16 10:19:58.756916] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1132452, clean_num=62914) [2024-08-16 10:19:58.761500] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF023-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:19:58.761555] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF023-0-0] [lt=29] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:19:58.761578] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF023-0-0] [lt=22] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=91015, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:19:58.761590] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF023-0-0] [lt=10][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509838, log_task={header:{begin_lsn:{lsn:18544219119}, end_lsn:{lsn:18544219241}, log_id:53509838, min_scn:{val:1723774798376439002, v:0}, max_scn:{val:1723774798376439002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544218997}, prev_proposal_id:7, committed_end_lsn:{lsn:18544219119}, data_checksum:1913811101, accum_checksum:867516980, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774798661348, freeze_ts:1723774798661348, submit_ts:1723774798661353, flushed_ts:1723774798661981, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:628}, fs_cb_begin_ts=1723774798761544, log_life_time=100196) [2024-08-16 10:19:58.762141] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=15] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1745) [2024-08-16 10:19:58.798645] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.798670] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:19:58.798682] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.798692] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.798707] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.830127] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=21] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=2610931, replace_num=31457) [2024-08-16 10:19:58.830157] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=27] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=10) [2024-08-16 10:19:58.842323] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:58.842435] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:58.842460] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:58.842467] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:58.843147] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:19:58.844154] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:58.850089] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.850110] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:58.850120] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.850129] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.850143] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.862243] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=707, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:19:58.875992] INFO [PALF] read (log_cache.cpp:100) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=51] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:58.876283] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [31754][T1_LockWaitMgr][T1][Y0-0000000000000000-0-0] [lt=50] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-08-16 10:19:58.898731] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.898751] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:58.898762] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.898771] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.898786] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.920136] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=29] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:19:58.942392] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:58.942500] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:19:58.942527] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1220) [2024-08-16 10:19:58.942541] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:58.942550] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:58.942555] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:58.944231] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=39][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:58.950132] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.950152] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:58.950163] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.950190] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=26] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.950205] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:19:58.951169] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=10] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:58.951209] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=29] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:58.951379] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:19:58.951410] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=23][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:19:58.951429] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=15][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:19:58.951437] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=7][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:19:58.952722] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7436E-0-0] [lt=41] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=7, avg_log_submit_to_flush_time=588, avg_log_submit_to_slide_time=1007) [2024-08-16 10:19:58.957422] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:58.957682] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=31] Wash time detail, (compute_wash_size_time=156, refresh_score_time=67, wash_time=16) [2024-08-16 10:19:58.960992] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1195366, clean_num=62914) [2024-08-16 10:19:58.961735] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF02F-0-0] [lt=22][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509840, log_task={header:{begin_lsn:{lsn:18544219363}, end_lsn:{lsn:18544219485}, log_id:53509840, min_scn:{val:1723774798376439004, v:0}, max_scn:{val:1723774798376439004, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544219241}, prev_proposal_id:7, committed_end_lsn:{lsn:18544219363}, data_checksum:1913811101, accum_checksum:2786674947, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774798861612, freeze_ts:1723774798861612, submit_ts:1723774798861617, flushed_ts:1723774798862301, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:684}, fs_cb_begin_ts=1723774798961726, log_life_time=100114) [2024-08-16 10:19:58.970985] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007C-000616D3CE4B9709-0-0] [lt=42][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:58.976278] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74370-0-0] [lt=44] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=606, avg_log_submit_to_slide_time=1151) [2024-08-16 10:19:58.998804] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:58.998827] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:19:58.998839] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:58.998848] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:58.998862] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.012094] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=38] gc stale ls task succ [2024-08-16 10:19:59.012228] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=4] gc stale ls task succ [2024-08-16 10:19:59.012265] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=21] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:19:59.012300] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=27] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:19:59.012316] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=14] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:19:59.016479] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:59.017116] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:59.024282] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C0B-0-0] [lt=24] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774799024224) [2024-08-16 10:19:59.030247] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=9] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=9) [2024-08-16 10:19:59.042473] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:59.042578] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:59.042592] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:59.042599] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:59.044304] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:59.049300] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=22] PNIO [ratelimit] time: 1723774799049298, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007623, add_bytes: 0 [2024-08-16 10:19:59.050230] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.050265] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=33] handle task succ(id={id:1260}) [2024-08-16 10:19:59.050275] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.050284] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.050298] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.051287] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=28] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:19:59.053795] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:19:59.054210] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80080-00061CCCC6B436C0-0-0] [lt=23][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:59.055575] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [2663][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=27] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,1,False,{False,},False,False,True}]) [2024-08-16 10:19:59.056336] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [31642][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=27] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-08-16 10:19:59.056699] INFO [STORAGE.TRANS] run1 (ob_black_list.cpp:194) [31842][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=24] ls blacklist refresh finish(cost_time=11673) [2024-08-16 10:19:59.062195] INFO [LIB] stat (utility.h:1197) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=34] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=41, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:19:59.062229] INFO [LIB] stat (utility.h:1197) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=33] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=6, this=0x7fa713668b28, extra_info="") [2024-08-16 10:19:59.062672] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=34, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:19:59.064823] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:19:59.064845] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:19:59.075120] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=20] PNIO [ratelimit] time: 1723774799075118, bytes: 1496436223249, bw: 0.160028 MB/s, add_ts: 1000827, add_bytes: 167940 [2024-08-16 10:19:59.075152] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85D4-0-0] [lt=28] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:19:59.098878] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.098901] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:59.098913] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.098923] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.098937] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.123397] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7D9-0-0] [lt=80][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:19:59.123992] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7D9-0-0] [lt=27] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:19:59.137349] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AAC-0-0] [lt=18] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:19:59.137395] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AAC-0-0] [lt=26] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774798754570001, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774798754570001, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:19:59.142552] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:59.142659] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:59.142675] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:59.142681] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:59.143258] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1342) [2024-08-16 10:19:59.143325] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=87] get_max_applied_scn(scn={val:18446744073709551615, v:3}, this={ls_id_:{id:1079}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:18446744073709551615, v:3}}) [2024-08-16 10:19:59.144386] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:59.150285] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.150305] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:19:59.150315] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.150324] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.150338] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.152026] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A9D-0-0] [lt=14] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:19:59.152055] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A9D-0-0] [lt=27] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:19:59.152104] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849A9D-0-0] [lt=9][errcode=-4389] log_task life cost too much time(palf_id=1262, self="192.168.0.128:2882", log_id=34459334, log_task={header:{begin_lsn:{lsn:8394842390}, end_lsn:{lsn:8394842512}, log_id:34459334, min_scn:{val:1723774798754570002, v:0}, max_scn:{val:1723774798754570002, v:0}, data_len:66, proposal_id:2, prev_lsn:{lsn:8394842268}, prev_proposal_id:2, committed_end_lsn:{lsn:8394842390}, data_checksum:2071192402, accum_checksum:1734453434, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774799051891, freeze_ts:1723774799051891, submit_ts:1723774799051896, flushed_ts:1723774799053865, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:1969}, fs_cb_begin_ts=1723774799152092, log_life_time=100201) [2024-08-16 10:19:59.152132] INFO [PALF] read (log_cache.cpp:100) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=71] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1342, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:59.161423] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:59.161691] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash time detail, (compute_wash_size_time=111, refresh_score_time=66, wash_time=16) [2024-08-16 10:19:59.165181] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1258280, clean_num=62914) [2024-08-16 10:19:59.175443] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85D6-0-0] [lt=32] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:19:59.176928] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21120][T1_L0_G0][T1][YB42C0A80080-00061CCCC241F690-0-0] [lt=17][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=8, len=105, pos=86, bitmap=NULL) [2024-08-16 10:19:59.183364] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=19] gc stale ls task succ [2024-08-16 10:19:59.198955] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.198982] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:19:59.198994] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.199004] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.199018] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.206851] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2520][T1002_TenantMet][T1002][Y0-0000000000000000-0-0] [lt=44] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:17374, free_obj_hold_cnt:593, allocator used:71436792, allocator total:73453184}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:1994, free_obj_hold_cnt:6, allocator used:131182000, allocator total:131182000}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():63488, full allocator used:1600, full allocator total:129024}, tablets_mem=204764208, tablets_mem_limit=1288490180, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:7, allocator used:17024, allocator total:65408}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:106, free_obj_hold_cnt:5057, allocator used:10243392, allocator total:10596096}, pending_cnt=0, wait_gc_count=0, tablet count=19368) [2024-08-16 10:19:59.230355] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=20] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=8) [2024-08-16 10:19:59.236324] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=18] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:59.236379] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=13] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:59.236400] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:59.236509] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:59.236837] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=45] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:59.237106] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:59.237328] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:19:59.242641] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:59.242754] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:59.242783] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:59.242791] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:59.243455] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=616, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:19:59.243498] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=29, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:19:59.243525] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:19:59.243980] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5B8-0-0] [lt=31] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:19:59.244000] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5B8-0-0] [lt=19] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1243, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:19:59.244476] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:59.248126] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE7E-0-0] [lt=24] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:19:59.248171] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE7E-0-0] [lt=27] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774798860610003, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774798860610003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:19:59.250360] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.250382] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:19:59.250392] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.250402] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.250413] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.261852] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF041-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:19:59.261894] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF041-0-0] [lt=28][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509843, log_task={header:{begin_lsn:{lsn:18544219729}, end_lsn:{lsn:18544219851}, log_id:53509843, min_scn:{val:1723774798877764002, v:0}, max_scn:{val:1723774798877764002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544219607}, prev_proposal_id:7, committed_end_lsn:{lsn:18544219729}, data_checksum:1913811101, accum_checksum:4055523706, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774799161797, freeze_ts:1723774799161797, submit_ts:1723774799161802, flushed_ts:1723774799162449, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:647}, fs_cb_begin_ts=1723774799261884, log_life_time=100087) [2024-08-16 10:19:59.262424] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=39] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=23307979, this={dio_aligned_buf:{buf_write_offset:1861, buf_padding_size:2235, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:40624691, truncate_used_ts:10122063}, log_block_size:67108864, dir_fd:156, io_fd:95}, aligned_buf_len=4096, aligned_buf="[", aligned_block_offset=23306240, buf_len=122, total_write_size_=15030, total_write_size_after_dio_=446464, ob_pwrite_used_ts=66584, count=105, each_pwrite_cost=634) [2024-08-16 10:19:59.274131] INFO [SQL.PC] inner_cache_evict (ob_ps_cache.cpp:788) [17389][T1002_PsCacheEv][T1002][Y0-0000000000000000-0-0] [lt=35] ps cache evict(tenant_id=1002, stmt_id_map_.size()=8, stmt_info_map_.size()=8, op.get_used_size()=35631, get_mem_high()=144955080, expired_stmt_ids.count()=0, closed_stmt_ids.count()=8) [2024-08-16 10:19:59.275872] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=16] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=551, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:19:59.275901] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:19:59.275929] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=15] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=33, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:19:59.275972] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=38, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:19:59.276238] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5BB-0-0] [lt=36] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:19:59.276264] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5BB-0-0] [lt=25] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1168, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:19:59.276826] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=22] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574620, offset=42142389}, end_cursor:ObLogCursor{file_id=65, log_id=18574621, offset=42142630}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751698050, len:1355, log_data_len:241, seq:18574620, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:59.277199] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=31] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574621, offset=42142630}, end_cursor:ObLogCursor{file_id=65, log_id=18574622, offset=42142879}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516a0050, len:1114, log_data_len:249, seq:18574621, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:59.278704] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA1-0-0] [lt=17] push gts task success(*task={:""}) [2024-08-16 10:19:59.290023] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA1-0-0] [lt=30] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:19:59.293027] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=26] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574622, offset=42142879}, end_cursor:ObLogCursor{file_id=65, log_id=18574623, offset=42143128}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516d8050, len:865, log_data_len:249, seq:18574622, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:59.293702] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=33] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574623, offset=42143128}, end_cursor:ObLogCursor{file_id=65, log_id=18574624, offset=42143377}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75170c050, len:616, log_data_len:249, seq:18574623, flush_finish:false, flush_ret:0}) [2024-08-16 10:19:59.299018] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.299038] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:59.299049] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.299059] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.299090] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.299498] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=15] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:19:59.299527] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=26] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=5021, mem_used=39515136) [2024-08-16 10:19:59.327290] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [2749][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=24] run rewrite rule refresh task(rule_mgr_->tenant_id_=1001) [2024-08-16 10:19:59.342711] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:59.342830] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:59.342846] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:59.342853] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:59.344553] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:59.350414] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.350440] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:19:59.350454] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.350490] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=34] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.350503] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.362030] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF047-0-0] [lt=10] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1623) [2024-08-16 10:19:59.362104] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF047-0-0] [lt=36] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=630, avg_log_submit_to_slide_time=91007) [2024-08-16 10:19:59.362142] INFO [PALF] read (log_cache.cpp:100) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=54] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1623, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:59.365633] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:59.365933] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=33] Wash time detail, (compute_wash_size_time=132, refresh_score_time=76, wash_time=16) [2024-08-16 10:19:59.369511] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1321194, clean_num=62914) [2024-08-16 10:19:59.369884] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=7] ====== traversal_flush timer task ====== [2024-08-16 10:19:59.370018] INFO [MDS] flush (mds_table_mgr.cpp:195) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=25] [FREEZE]no need do flush cause min_rec_scn is larger than freezing scn(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1079}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:32, ordered_tablets:{{tablet_id:{id:208204}, rec_scn:{val:1723723445809886004, v:0}}, {tablet_id:{id:1152921504606884836}, rec_scn:{val:1723723445809886006, v:0}}, {tablet_id:{id:1152921504606884831}, rec_scn:{val:1723723445809886007, v:0}}, {tablet_id:{id:208210}, rec_scn:{val:1723723445809886008, v:0}}, {tablet_id:{id:1152921504606884610}, rec_scn:{val:1723723445809886009, v:0}}, {tablet_id:{id:1152921504606895583}, rec_scn:{val:1723723445809886011, v:0}}, {tablet_id:{id:1152921504606884840}, rec_scn:{val:1723723445809886012, v:0}}, {tablet_id:{id:1152921504606884615}, rec_scn:{val:1723723445809886013, v:0}}, {tablet_id:{id:1152921504606903842}, rec_scn:{val:1723723446311099001, v:0}}, {tablet_id:{id:208206}, rec_scn:{val:1723723446311099002, v:0}}, {tablet_id:{id:208211}, rec_scn:{val:1723723446311099003, v:0}}, {tablet_id:{id:1152921504606884595}, rec_scn:{val:1723723446311099005, v:0}}, {tablet_id:{id:219872}, rec_scn:{val:1723723446311099006, v:0}}, {tablet_id:{id:208218}, rec_scn:{val:1723723446311099007, v:0}}, {tablet_id:{id:1152921504606884825}, rec_scn:{val:1723723446311099008, v:0}}, {tablet_id:{id:208208}, rec_scn:{val:1723723446311099010, v:0}}, {tablet_id:{id:1152921504606884606}, rec_scn:{val:1723723446311099011, v:0}}, {tablet_id:{id:1152921504606884588}, rec_scn:{val:1723723446311099012, v:0}}, {tablet_id:{id:1152921504606884580}, rec_scn:{val:1723723446311099013, v:0}}, {tablet_id:{id:1152921504606895579}, rec_scn:{val:1723723446311099015, v:0}}, {tablet_id:{id:1152921504606884601}, rec_scn:{val:1723723446311099016, v:0}}, {tablet_id:{id:1152921504606884609}, rec_scn:{val:1723723446311099017, v:0}}, {tablet_id:{id:1152921504606884618}, rec_scn:{val:1723723446311099018, v:0}}, {tablet_id:{id:1152921504606884839}, rec_scn:{val:1723723446311099019, v:0}}, {tablet_id:{id:1152921504606884599}, rec_scn:{val:1723723446311099021, v:0}}, {tablet_id:{id:1152921504606884616}, rec_scn:{val:1723723446311099022, v:0}}, {tablet_id:{id:1152921504606895582}, rec_scn:{val:1723723446812238001, v:0}}, {tablet_id:{id:1152921504606884581}, rec_scn:{val:1723723446812238002, v:0}}, {tablet_id:{id:1152921504606884590}, rec_scn:{val:1723723446812238003, v:0}}, {tablet_id:{id:1152921504606884827}, rec_scn:{val:1723723446812238005, v:0}}, {tablet_id:{id:1152921504606914006}, rec_scn:{val:1723723446812238006, v:0}}, {tablet_id:{id:214096}, rec_scn:{val:1723723446812238007, v:0}}, }, max_consequent_callbacked_scn={val:1723774798877764004, v:0}, *this={this:0x7fa6fa246510, is_inited:true, freezing_scn:{val:1721997282211836001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774797651358002, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:19:59.371797] INFO [MDS] flush (mds_table_mgr.cpp:195) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=141] [FREEZE]no need do flush cause min_rec_scn is larger than freezing scn(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1260}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:32, ordered_tablets:{{tablet_id:{id:1152921504606868225}, rec_scn:{val:1723723302639709002, v:0}}, {tablet_id:{id:1152921504606912559}, rec_scn:{val:1723723302639709003, v:0}}, {tablet_id:{id:1152921504606871965}, rec_scn:{val:1723723302639709005, v:0}}, {tablet_id:{id:212886}, rec_scn:{val:1723723302639709006, v:0}}, {tablet_id:{id:1152921504606858436}, rec_scn:{val:1723723302639709007, v:0}}, {tablet_id:{id:212255}, rec_scn:{val:1723723302639709009, v:0}}, {tablet_id:{id:1152921504606876671}, rec_scn:{val:1723723302639709010, v:0}}, {tablet_id:{id:1152921504606910457}, rec_scn:{val:1723723302639709012, v:0}}, {tablet_id:{id:214006}, rec_scn:{val:1723723302639709013, v:0}}, {tablet_id:{id:1152921504606912491}, rec_scn:{val:1723723302639709014, v:0}}, {tablet_id:{id:1152921504606890057}, rec_scn:{val:1723723303141598001, v:0}}, {tablet_id:{id:1152921504606848015}, rec_scn:{val:1723723303141598002, v:0}}, {tablet_id:{id:1152921504606847616}, rec_scn:{val:1723723303141598004, v:0}}, {tablet_id:{id:1152921504606913427}, rec_scn:{val:1723723303141598005, v:0}}, {tablet_id:{id:1152921504606915480}, rec_scn:{val:1723723303141598006, v:0}}, {tablet_id:{id:1152921504606889072}, rec_scn:{val:1723723303141598008, v:0}}, {tablet_id:{id:1152921504606915220}, rec_scn:{val:1723723303141598009, v:0}}, {tablet_id:{id:1152921504606866874}, rec_scn:{val:1723723303141598010, v:0}}, {tablet_id:{id:200529}, rec_scn:{val:1723723303141598012, v:0}}, {tablet_id:{id:1152921504606870508}, rec_scn:{val:1723723303141598013, v:0}}, {tablet_id:{id:1152921504606903696}, rec_scn:{val:1723723303141598015, v:0}}, {tablet_id:{id:1152921504606896782}, rec_scn:{val:1723723303647138001, v:0}}, {tablet_id:{id:1152921504606898434}, rec_scn:{val:1723723303647138002, v:0}}, {tablet_id:{id:1152921504606899144}, rec_scn:{val:1723723303647138004, v:0}}, {tablet_id:{id:1152921504606908791}, rec_scn:{val:1723723303829693001, v:0}}, {tablet_id:{id:1152921504606871378}, rec_scn:{val:1723723303829693002, v:0}}, {tablet_id:{id:1152921504606870897}, rec_scn:{val:1723723303829693003, v:0}}, {tablet_id:{id:1152921504606877002}, rec_scn:{val:1723723303829693004, v:0}}, {tablet_id:{id:1152921504606886744}, rec_scn:{val:1723723303829693006, v:0}}, {tablet_id:{id:1152921504606875352}, rec_scn:{val:1723723303829693007, v:0}}, {tablet_id:{id:204192}, rec_scn:{val:1723723303829693009, v:0}}, {tablet_id:{id:1152921504606897887}, rec_scn:{val:1723723304142080001, v:0}}, }, max_consequent_callbacked_scn={val:1723774798918444003, v:0}, *this={this:0x7fa6bd53e510, is_inited:true, freezing_scn:{val:1723690100831412000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1260}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202608224000, v:0}, clog_base_lsn:{lsn:8723619840}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774797651358002, v:0}, tablet_change_checkpoint_scn:{val:1723719123293042000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723639313794159001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1260, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:19:59.372396] INFO [MDS] flush (mds_table_mgr.cpp:172) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=118] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1261}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x7fa6fe204510, is_inited:true, freezing_scn:{val:1723752041922970000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1261}, ls_create_status:1, clog_checkpoint_scn:{val:1723752188579788000, v:0}, clog_base_lsn:{lsn:32545812480}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774797651358002, v:0}, tablet_change_checkpoint_scn:{val:1723719128263216001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723617733349446001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:4, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1261}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1261, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:19:59.373929] INFO [MDS] flush (mds_table_mgr.cpp:195) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=50] [FREEZE]no need do flush cause min_rec_scn is larger than freezing scn(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1262}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:32, ordered_tablets:{{tablet_id:{id:209134}, rec_scn:{val:1723723207973113002, v:0}}, {tablet_id:{id:1152921504606869956}, rec_scn:{val:1723723207973113003, v:0}}, {tablet_id:{id:1152921504606871991}, rec_scn:{val:1723723207973113005, v:0}}, {tablet_id:{id:1152921504606868203}, rec_scn:{val:1723723207973113006, v:0}}, {tablet_id:{id:1152921504606857189}, rec_scn:{val:1723723207973113007, v:0}}, {tablet_id:{id:1152921504606903903}, rec_scn:{val:1723723207973113009, v:0}}, {tablet_id:{id:205824}, rec_scn:{val:1723723207973113010, v:0}}, {tablet_id:{id:207632}, rec_scn:{val:1723723207973113011, v:0}}, {tablet_id:{id:1152921504606888997}, rec_scn:{val:1723723207973113013, v:0}}, {tablet_id:{id:214372}, rec_scn:{val:1723723207973113014, v:0}}, {tablet_id:{id:1152921504606915317}, rec_scn:{val:1723723207973113015, v:0}}, {tablet_id:{id:1152921504606875134}, rec_scn:{val:1723723207973113016, v:0}}, {tablet_id:{id:1152921504606876196}, rec_scn:{val:1723723208473765001, v:0}}, {tablet_id:{id:221202}, rec_scn:{val:1723723208473765002, v:0}}, {tablet_id:{id:1152921504606888619}, rec_scn:{val:1723723208473765003, v:0}}, {tablet_id:{id:1152921504606898978}, rec_scn:{val:1723723208473765005, v:0}}, {tablet_id:{id:1152921504606897298}, rec_scn:{val:1723723208473765006, v:0}}, {tablet_id:{id:1152921504606918529}, rec_scn:{val:1723723208473765007, v:0}}, {tablet_id:{id:1152921504606872365}, rec_scn:{val:1723723208473765009, v:0}}, {tablet_id:{id:1152921504606898136}, rec_scn:{val:1723723208473765010, v:0}}, {tablet_id:{id:210672}, rec_scn:{val:1723723208473765011, v:0}}, {tablet_id:{id:209037}, rec_scn:{val:1723723208473765013, v:0}}, {tablet_id:{id:1152921504606858115}, rec_scn:{val:1723723208473765014, v:0}}, {tablet_id:{id:1152921504606896677}, rec_scn:{val:1723723208473765015, v:0}}, {tablet_id:{id:1152921504606896744}, rec_scn:{val:1723723208473765016, v:0}}, {tablet_id:{id:218140}, rec_scn:{val:1723723208473765018, v:0}}, {tablet_id:{id:1152921504606856887}, rec_scn:{val:1723723208473765019, v:0}}, {tablet_id:{id:1152921504606856837}, rec_scn:{val:1723723208473765020, v:0}}, {tablet_id:{id:1152921504606867226}, rec_scn:{val:1723723208973569001, v:0}}, {tablet_id:{id:1152921504606912513}, rec_scn:{val:1723723208973569002, v:0}}, {tablet_id:{id:1152921504606856874}, rec_scn:{val:1723723208973569003, v:0}}, {tablet_id:{id:1152921504606896355}, rec_scn:{val:1723723208973569004, v:0}}, }, max_consequent_callbacked_scn={val:1723774799254701000, v:0}, *this={this:0x7fa6be4fc510, is_inited:true, freezing_scn:{val:1723656976361279000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774797651358002, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:19:59.374032] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:336) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=98] succeed to traversal_flush(ret=0, ls_cnt=4) [2024-08-16 10:19:59.375406] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85DA-0-0] [lt=16] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1342, avg_group_log_size=122) [2024-08-16 10:19:59.376998] INFO [STORAGE] schedule_all_tablets_minor (ob_tenant_tablet_scheduler.cpp:511) [2569][T1002_MergeLoop][T1002][YB42C0A80080-00061CCC96523AAD-0-0] [lt=23] start schedule all tablet minor merge(minor_ls_tablet_iter_={ls_idx:-1, ls_ids:[{id:1079}, {id:1260}, {id:1261}, {id:1262}], tablet_idx:0, tablet_ids_.count():0, schedule_tablet_cnt:0, max_batch_tablet_cnt:50000, report_scn_flag:false}) [2024-08-16 10:19:59.383895] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=23] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:19:59.399101] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.399125] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:19:59.399135] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.399145] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.399186] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=33] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.405763] INFO pn_ratelimit (group.c:551) [31557][IngressService][T0][Y0-0000000000000000-0-0] [lt=23] PNIO set ratelimit as 9223372036854775807 bytes/s, grp_id=2 [2024-08-16 10:19:59.421127] INFO [STORAGE.TRANS] alloc (ob_ts_response_handler.cpp:108) [31439][pnio1][T0][YB42C0A80080-00061CCCCD6206A1-0-0] [lt=22] ts response task statistics(alloc_count=170000366, free_count=85000182) [2024-08-16 10:19:59.425098] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=20] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:19:59.425216] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=56] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:19:59.430466] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=26] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=7) [2024-08-16 10:19:59.442778] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] tx gc loop thread is running(MTL_ID()=1002) [2024-08-16 10:19:59.443983] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774797651358002, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:19:59.444123] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=136] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:59.444639] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:59.445088] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1260}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202608224000, v:0}, clog_base_lsn:{lsn:8723619840}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774797651358002, v:0}, tablet_change_checkpoint_scn:{val:1723719123293042000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723639313794159001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1260, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:19:59.445235] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=69] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:59.446248] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1261}, ls_create_status:1, clog_checkpoint_scn:{val:1723752188579788000, v:0}, clog_base_lsn:{lsn:32545812480}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774797651358002, v:0}, tablet_change_checkpoint_scn:{val:1723719128263216001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723617733349446001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:4, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1261}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1261, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:19:59.446317] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=66] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:59.447568] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774797651358002, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:19:59.447646] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=76] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:59.450507] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.450533] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:19:59.450566] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=29] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.450585] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.450606] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.462027] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF04D-0-0] [lt=22][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509846, log_task={header:{begin_lsn:{lsn:18544220376}, end_lsn:{lsn:18544220498}, log_id:53509846, min_scn:{val:1723774798877764005, v:0}, max_scn:{val:1723774798877764005, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544219973}, prev_proposal_id:7, committed_end_lsn:{lsn:18544220376}, data_checksum:1913811101, accum_checksum:2487639506, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774799362005, freeze_ts:1723774799362005, submit_ts:1723774799362012, flushed_ts:1723774799362726, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:714}, fs_cb_begin_ts=1723774799462014, log_life_time=100009) [2024-08-16 10:19:59.484134] INFO [CLOG] is_in_sync (ob_log_handler.cpp:546) [31817][T1_L5_G0][T1002][YB42C0A8007D-00061CCCA5024D1C-0-0] [lt=20] is_in_sync(ret=0, id=1079, is_log_sync=true, is_need_rebuild=false) [2024-08-16 10:19:59.484226] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007D-00061CCCA5024D1C-0-0] [lt=25][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:59.487384] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7DD-0-0] [lt=22] [RS_MGR] force set rs list(master_rs="192.168.0.126:2882") [2024-08-16 10:19:59.487421] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7DD-0-0] [lt=28] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"192.168.0.128:2882", server_id:16, rs_addr:"192.168.0.126:2882", rs_server_status:1, epoch_id:11}, hb_response={zone:"zone2", server:"192.168.0.128:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1720514727760575, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=39) [2024-08-16 10:19:59.499214] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.499247] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=31] handle task succ(id={id:1260}) [2024-08-16 10:19:59.499257] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.499266] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.499279] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.501902] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [2672][T1001_PalfGC][T1001][Y0-0000000000000000-0-0] [lt=33] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):2150, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):2150, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}) [2024-08-16 10:19:59.504510] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80083-000616E44D0F990C-0-0] [lt=24][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:19:59.509111] INFO [CLOG] push_log_replay_task (ob_replay_status.cpp:1140) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=23] submit pre barrier log success(log_type=17, ls_id={id:1261}, lsn={lsn:32601561366}, scn={val:1723774799360164002, v:0}, is_pre_barrier=true, is_post_barrier=false, log_size=24) [2024-08-16 10:19:59.509099] INFO [CLOG] do_replay_task_ (ob_log_replay_service.cpp:901) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=18] wait for barrier(ret=-4023, replay_status={ls_id_:{id:1261}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:5, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774799508962, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:344950, ret_code_:0}}, next_to_submit_lsn_:{lsn:32601561310}, next_to_submit_scn_:{val:1723774799360164002, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516552080113001, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd805000, next_round_pread_size:3678208, curr_read_pos:112, curr_read_buf_start_pos:0, curr_read_buf_end_pos:112, log_storage_:{IteratorStorage:{start_lsn:{lsn:32601561310}, end_lsn:{lsn:32601561422}, read_buf:{buf_len_:3682304, buf_:0x7fa6cd805000}, block_size:67104768, log_storage_:0x7fa6fe299470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774799360164002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:24, scn_:{val:1723774799360164002, v:0}, data_checksum:1538219805, flag:0}}, init_mode_version:0, accumulate_checksum:424179756, curr_entry_is_padding:0, padding_entry_size:56, padding_entry_scn:{val:1723774799360164002, v:0}}}}}, replay_task={ls_id_:{id:1261}, log_type:17, log_type:"RESERVED_SNAPSHOT", lsn_:{lsn:32601561366}, scn_:{val:1723774799360164002, v:0}, is_pre_barrier_:true, is_post_barrier_:false, read_log_size_:24, replay_hint_:1723774799506528, first_handle_ts_:1723774799509095, replay_cost_:8243121546482100256, retry_cost_:4210989222874735990, read_log_buf_:0x7fa5a4177380, decompression_buf_:null, has_decompressed_:false, decompressed_log_size_:0, base_header_len_:16}) [2024-08-16 10:19:59.509332] INFO [STORAGE] inner_update_reserved_snapshot (ob_ls_reserved_snapshot_mgr.cpp:288) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=28] success to update reserved snapshot(ret=0, ls_id={id:1261}, min_reserved_snapshot_=1723772990629965000) [2024-08-16 10:19:59.509379] INFO [CLOG] do_replay_task_ (ob_log_replay_service.cpp:919) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=43] pre barrier log replay succ(replay_task={ls_id_:{id:1261}, log_type:17, log_type:"RESERVED_SNAPSHOT", lsn_:{lsn:32601561366}, scn_:{val:1723774799360164002, v:0}, is_pre_barrier_:true, is_post_barrier_:false, read_log_size_:24, replay_hint_:1723774799506528, first_handle_ts_:1723774799509309, replay_cost_:67, retry_cost_:4210989222874735990, read_log_buf_:0x7fa5a4177390, decompression_buf_:null, has_decompressed_:false, decompressed_log_size_:0, base_header_len_:16}, replay_status={ls_id_:{id:1261}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774799508962, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:344950, ret_code_:0}}, next_to_submit_lsn_:{lsn:32601561422}, next_to_submit_scn_:{val:1723774799360164003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516552080113001, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd805000, next_round_pread_size:3678208, curr_read_pos:112, curr_read_buf_start_pos:0, curr_read_buf_end_pos:112, log_storage_:{IteratorStorage:{start_lsn:{lsn:32601561310}, end_lsn:{lsn:32601561422}, read_buf:{buf_len_:3682304, buf_:0x7fa6cd805000}, block_size:67104768, log_storage_:0x7fa6fe299470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774799360164002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:24, scn_:{val:1723774799360164002, v:0}, data_checksum:1538219805, flag:0}}, init_mode_version:0, accumulate_checksum:424179756, curr_entry_is_padding:0, padding_entry_size:56, padding_entry_scn:{val:1723774799360164002, v:0}}}}}, replay_queue_idx=0, ret=0) [2024-08-16 10:19:59.525422] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=28] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=1, size_used=6897, mem_used=12478464) [2024-08-16 10:19:59.532884] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=25] start do ls ha handler(ls_id_array_=[{id:1079}, {id:1261}, {id:1262}, {id:1260}]) [2024-08-16 10:19:59.542870] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:59.542997] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:59.543018] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:59.543030] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:59.544720] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:59.550545] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.550566] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:59.550576] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.550588] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.550603] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.551606] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 362.00) [2024-08-16 10:19:59.551631] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=19] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:59.551699] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=32] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.99, rt: 415.75) [2024-08-16 10:19:59.551716] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=13] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SHARED_BLOCK_ReaderWriter_IO, mode: read, size: 4915.20, iops: 4.98, rt: 310.80) [2024-08-16 10:19:59.551722] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:19:59.551745] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=18] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551756] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551768] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=11] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551775] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551780] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551787] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551796] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551801] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551805] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551809] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551813] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551818] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551822] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551828] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551836] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.551841] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:19:59.552292] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AA5-0-0] [lt=63] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:19:59.554429] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=47] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=830, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:19:59.554458] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=21, stat_interval=1000000, avg (count/cost)=31, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:19:59.554513] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=35, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:19:59.570019] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:59.570311] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Wash time detail, (compute_wash_size_time=130, refresh_score_time=83, wash_time=40) [2024-08-16 10:19:59.573458] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=15] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1384108, clean_num=62914) [2024-08-16 10:19:59.584333] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=21] ====== check clog disk timer task ====== [2024-08-16 10:19:59.584358] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=21] get_disk_usage(ret=0, capacity(MB):=2150, used(MB):=0) [2024-08-16 10:19:59.584372] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=8] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=676457349, need_update_checkpoint_scn=false) [2024-08-16 10:19:59.585474] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=19] ====== check clog disk timer task ====== [2024-08-16 10:19:59.585500] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=23] get_disk_usage(ret=0, capacity(MB):=0, used(MB):=0) [2024-08-16 10:19:59.585516] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=11] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=0, need_update_checkpoint_scn=false) [2024-08-16 10:19:59.591055] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=9][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:19:59.593219] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=53] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:19:59.593312] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=22][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:19:59.593332] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=19][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:19:59.593361] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=28] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:19:59.599294] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.599319] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:19:59.599328] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.599336] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.599352] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.606963] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=15] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:19:59.610251] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [31774][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=10] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:19:59.610276] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:230) [31774][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=18] [tabletchange] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12637) [2024-08-16 10:19:59.616987] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=34] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60673, mem_used=137263104) [2024-08-16 10:19:59.620541] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [25973][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=42] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:19:59.630592] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=31] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=6) [2024-08-16 10:19:59.642924] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:59.643045] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:59.643062] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:59.643100] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=36] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:59.644206] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5C6-0-0] [lt=32] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=5, avg_log_gen_to_submit_time=61, avg_log_submit_to_flush_time=682, avg_log_submit_to_slide_time=1173) [2024-08-16 10:19:59.644825] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=29][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:59.650609] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.650644] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=33] handle task succ(id={id:1260}) [2024-08-16 10:19:59.650655] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.650665] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.650679] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.652238] INFO [STORAGE] runTimerTask (ob_tenant_tablet_scheduler.cpp:135) [2569][T1002_MergeLoop][T1002][YB42C0A80080-00061CCC96523AAD-0-0] [lt=28] MergeLoopTask(cost_ts=275265) [2024-08-16 10:19:59.652411] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AA7-0-0] [lt=23][errcode=-4389] log_task life cost too much time(palf_id=1262, self="192.168.0.128:2882", log_id=34459339, log_task={header:{begin_lsn:{lsn:8394842991}, end_lsn:{lsn:8394843113}, log_id:34459339, min_scn:{val:1723774799254701002, v:0}, max_scn:{val:1723774799254701002, v:0}, data_len:66, proposal_id:2, prev_lsn:{lsn:8394842869}, prev_proposal_id:2, committed_end_lsn:{lsn:8394842991}, data_checksum:2071192402, accum_checksum:3909097291, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774799552286, freeze_ts:1723774799552286, submit_ts:1723774799552326, flushed_ts:1723774799554473, gen_to_freeze cost time:0, gen_to_submit cost time:40, submit_to_flush cost time:2147}, fs_cb_begin_ts=1723774799652398, log_life_time=100112) [2024-08-16 10:19:59.653386] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73593-0-0] [lt=38] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=15, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:19:59.653411] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73593-0-0] [lt=24] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=20879, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:19:59.699367] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.699393] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:19:59.699407] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.699420] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.699442] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.736922] INFO [COMMON] generate_mod_stat_task (memory_dump.cpp:218) [31418][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=21] task info(*task={type_:2, dump_all_:false, p_context_:null, slot_idx_:0, dump_tenant_ctx_:false, tenant_id_:0, ctx_id_:0, p_chunk_:null}) [2024-08-16 10:19:59.736953] INFO [COMMON] handle (memory_dump.cpp:498) [31418][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=24] handle dump task(task={type_:2, dump_all_:false, p_context_:null, slot_idx_:0, dump_tenant_ctx_:false, tenant_id_:0, ctx_id_:0, p_chunk_:null}) [2024-08-16 10:19:59.737100] INFO [COMMON] update_check_range (ob_sql_mem_leak_checker.cpp:62) [31418][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=22] update_check_range(min_check_version=324702, max_check_version=324703, global_version=324704) [2024-08-16 10:19:59.742998] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:59.743104] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=43, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:19:59.743141] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:19:59.743152] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:59.743159] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:59.743167] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:59.744875] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:59.750678] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.750696] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] handle task succ(id={id:1260}) [2024-08-16 10:19:59.750705] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.750713] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.750727] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.759262] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=24] LogLoopThread round_cost_time(us)(round_cost_time=20) [2024-08-16 10:19:59.762248] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF05F-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:19:59.762304] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF05F-0-0] [lt=28] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=9, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:19:59.762320] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF05F-0-0] [lt=15] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=91011, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:19:59.762328] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF05F-0-0] [lt=6][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509849, log_task={header:{begin_lsn:{lsn:18544220742}, end_lsn:{lsn:18544220864}, log_id:53509849, min_scn:{val:1723774799378452002, v:0}, max_scn:{val:1723774799378452002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544220620}, prev_proposal_id:7, committed_end_lsn:{lsn:18544220742}, data_checksum:1913811101, accum_checksum:4194442125, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774799662064, freeze_ts:1723774799662064, submit_ts:1723774799662092, flushed_ts:1723774799662653, gen_to_freeze cost time:0, gen_to_submit cost time:28, submit_to_flush cost time:561}, fs_cb_begin_ts=1723774799762294, log_life_time=100230) [2024-08-16 10:19:59.762968] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1623) [2024-08-16 10:19:59.765013] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=26] ====== tenant freeze timer task ====== [2024-08-16 10:19:59.765051] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=24][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:19:59.768176] INFO [STORAGE.TRANS] rollback_tx (ob_tx_api.cpp:339) [2568][T1002_ArcSrv][T1001][YB42C0A80080-00061CCC9251D0C5-0-0] [lt=53] rollback tx(ret=0, *this={is_inited_:true, tenant_id_:1001, this:0x7fa6e0004030}, tx={this:0x7fa704c14d80, tx_id:{txid:283711674}, state:7, addr:"192.168.0.128:2882", tenant_id:1001, session_id:3225602455, assoc_session_id:3225602455, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:1723774799757379, op_sn:3, alloc_ts:1723774799756597, active_ts:1723774799756597, commit_ts:-1, finish_ts:1723774799767313, timeout_us:29999984, lock_timeout_us:-1, expire_ts:1723774829756581, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:1713182723, cluster_version:17180000768, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:-6002, commit_expire_ts:0, commit_task_.is_registered():false, ref:2}) [2024-08-16 10:19:59.769117] INFO handle (memory_dump.cpp:626) [31418][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=14] statistics: tenant_cnt: 5, max_chunk_cnt: 524288 tenant_id ctx_id chunk_cnt label_cnt segv_cnt 1 0 202 180 0 1 5 22 5 0 1 7 1 2 0 1 8 54 1 0 1 12 8 3 0 1 13 5 1 0 1 14 1 1 0 1 20 1 1 0 500 0 113 256 0 500 7 13 4 0 500 8 55 1 0 500 9 2 1 0 500 10 6 2 0 500 16 3 1 0 500 17 5 7 0 500 22 54 58 0 500 23 27 11 0 508 0 7 13 0 508 8 13 1 0 1001 0 77 144 0 1001 5 23 5 0 1001 8 41 1 0 1001 12 1 1 0 1001 14 1 1 0 1001 16 1 1 0 1002 0 482 237 0 1002 2 138 1 0 1002 4 2 1 0 1002 5 129 5 0 1002 8 68 1 0 1002 12 108 3 0 1002 13 21 1 0 1002 14 1 2 0 1002 15 1 1 0 1002 18 30 3 0 1002 20 20 1 0 cost_time: 32052 [2024-08-16 10:19:59.773890] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:59.774182] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=115, refresh_score_time=65, wash_time=16) [2024-08-16 10:19:59.777605] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1447022, clean_num=62914) [2024-08-16 10:19:59.799451] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.799485] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=32] handle task succ(id={id:1260}) [2024-08-16 10:19:59.799496] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.799506] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.799520] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.807850] INFO [SERVER] reload_tenant_task (ob_tenant_tablet_ttl_mgr.cpp:1297) [5766][T1002_TTLTablet][T1002][YB42C0A80080-00061CCC98B6C840-0-0] [lt=39] finish reload tenant task(local_tenant_task_={tenant_id:1002, task_id:-1, is_usr_trigger:false, need_check:false, is_dirty:false, ttl_continue:true, rsp_time:-1, state:4, is_reused:true}, tenant_task={gmt_create:0, gmt_modified:0, tenant_id:18446744073709551615, table_id:18446744073709551615, tablet_id:18446744073709551615, task_id:0, task_start_time:0, task_update_time:0, trigger_type:0, status:19, ttl_del_cnt:0, max_version_del_cnt:0, scan_cnt:0, row_key:"", ret_code:"OB_SUCCESS"}, is_leader=false) [2024-08-16 10:19:59.830696] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=22] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=5) [2024-08-16 10:19:59.843094] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:59.843207] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:59.843224] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:59.843232] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:59.843984] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1211) [2024-08-16 10:19:59.844958] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:59.850749] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.850767] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] handle task succ(id={id:1260}) [2024-08-16 10:19:59.850777] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.850844] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=65] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.850860] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.859099] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:255) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E7D-0-0] [lt=14] [REFRESH_SCHEMA] start to process schema refresh task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:19:59.859103] INFO [SERVER] try_reload_schema (ob_server_schema_updater.cpp:440) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=15] schedule fetch new schema task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:19:59.859123] INFO [SERVER] do_heartbeat_event (ob_heartbeat.cpp:268) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=18] try reload schema success(schema_version=1722933080671200, refresh_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_ret=0) [2024-08-16 10:19:59.859129] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:280) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E7D-0-0] [lt=26] [REFRESH_SCHEMA] local schema info is newer, no need to refresh schema(ret=0, ret="OB_SUCCESS", local_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:19:59.863090] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=564, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:19:59.869709] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2752][T1001_TenantMet][T1001][Y0-0000000000000000-0-0] [lt=49] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=0, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:19:59.872011] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [2782][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=9] ====== checkpoint timer task ====== [2024-08-16 10:19:59.872035] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [2782][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=19] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:19:59.899562] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.899586] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:19:59.899599] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.899610] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.899625] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.921329] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=22] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:19:59.943147] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:19:59.943266] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1211, avg_group_log_size=121) [2024-08-16 10:19:59.943293] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1211) [2024-08-16 10:19:59.943307] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:19:59.943313] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:19:59.943318] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:19:59.945031] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:19:59.950819] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.950840] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:19:59.950850] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.950860] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.950871] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:19:59.951438] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=11] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:59.951474] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=24] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:59.951631] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=22][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:19:59.951654] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=17][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:19:59.951693] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=34][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:19:59.951717] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=22][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:19:59.953585] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7359C-0-0] [lt=35] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=10, avg_log_submit_to_flush_time=953, avg_log_submit_to_slide_time=20875) [2024-08-16 10:19:59.961528] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=9] ====== check clog disk timer task ====== [2024-08-16 10:19:59.961572] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=37] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:19:59.961601] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=14] cannot_recycle_log_size statistics(cannot_recycle_log_size=391635089, threshold=6088116142, need_update_checkpoint_scn=false) [2024-08-16 10:19:59.962664] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF06B-0-0] [lt=32][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509851, log_task={header:{begin_lsn:{lsn:18544220986}, end_lsn:{lsn:18544221108}, log_id:53509851, min_scn:{val:1723774799746746001, v:0}, max_scn:{val:1723774799746746001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544220864}, prev_proposal_id:7, committed_end_lsn:{lsn:18544220986}, data_checksum:1913811101, accum_checksum:2453812184, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774799862479, freeze_ts:1723774799862479, submit_ts:1723774799862484, flushed_ts:1723774799863132, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:648}, fs_cb_begin_ts=1723774799962653, log_life_time=100174) [2024-08-16 10:19:59.976750] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74390-0-0] [lt=49] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=676, avg_log_submit_to_slide_time=1070) [2024-08-16 10:19:59.976774] INFO [PALF] read (log_cache.cpp:100) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1445, hit_cnt=12, read_cnt=12, hit rate=1.000000000000000000e+00) [2024-08-16 10:19:59.978054] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:19:59.978398] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=52] Wash time detail, (compute_wash_size_time=139, refresh_score_time=114, wash_time=23) [2024-08-16 10:19:59.981765] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=23] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1509936, clean_num=62914) [2024-08-16 10:19:59.999636] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:19:59.999659] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:19:59.999671] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:19:59.999682] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:19:59.999695] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.012169] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=22] gc stale ls task succ [2024-08-16 10:20:00.012384] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=6] gc stale ls task succ [2024-08-16 10:20:00.012426] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=32] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:00.012442] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=10] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:00.012446] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=3] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:00.016209] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=39] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:00.016952] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:00.025048] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C0F-0-0] [lt=21] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774800024997) [2024-08-16 10:20:00.030836] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=41] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=4) [2024-08-16 10:20:00.043238] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:00.043364] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:00.043384] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:00.043392] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:00.043699] INFO [PALF] runTimerTask (block_gc_timer_task.cpp:101) [31651][T1_PalfGC][T1][Y0-0000000000000000-0-0] [lt=28] BlockGCTimerTask success(ret=0, cost_time_us=17, palf_env_impl_={IPalfEnvImpl:{IPalfEnvImpl:"Dummy"}, self:"192.168.0.128:2882", log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1", disk_options_wrapper:{disk_opts_for_stopping_writing:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}, log_alloc_mgr_:{flying_log_task:0, flying_meta_task:0}}) [2024-08-16 10:20:00.045113] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:00.050905] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.050931] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:20:00.050942] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.050965] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.050980] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.052138] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=23] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:00.056908] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=22] PNIO [ratelimit] time: 1723774800056905, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007607, add_bytes: 0 [2024-08-16 10:20:00.062743] INFO [LIB] stat (utility.h:1197) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=13] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=64, stat_interval=1000000, avg (count/cost)=77, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:00.062768] INFO [LIB] stat (utility.h:1197) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=24] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=64, stat_interval=1000000, avg (count/cost)=5, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:00.063404] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=31, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:20:00.064903] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:00.064926] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:00.075896] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=29] PNIO [ratelimit] time: 1723774800075894, bytes: 1496436344909, bw: 0.115934 MB/s, add_ts: 1000776, add_bytes: 121660 [2024-08-16 10:20:00.075947] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85E8-0-0] [lt=53] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:00.083887] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2747][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=29] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:20:00.086848] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [2566][T1002_ArcSender][T1002][YB42C0A80080-00061CCC9231D0C5-0-0] [lt=18][errcode=-4018] try free send task failed(ret=-4018) [2024-08-16 10:20:00.099744] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.099768] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:20:00.099780] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.099790] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.099805] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.105619] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [2783][T1001_Flush][T1001][Y0-0000000000000000-0-0] [lt=18] ====== traversal_flush timer task ====== [2024-08-16 10:20:00.105662] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:338) [2783][T1001_Flush][T1001][Y0-0000000000000000-0-0] [lt=23] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:20:00.106913] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80085-00061CCD1E636C12-0-0] [lt=22][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:00.126464] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DA-0-0] [lt=110][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:00.127022] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DA-0-0] [lt=26] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:00.143319] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:00.143451] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=41] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:00.143468] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:00.143475] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:00.144204] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1211) [2024-08-16 10:20:00.145184] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:134) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22] ObStandbyTimestampService state(*this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:20:00.145214] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=28][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:00.150980] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.151004] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:00.151017] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.151030] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.151068] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=28] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.152564] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AAD-0-0] [lt=33] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:00.152601] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AAD-0-0] [lt=22] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774799756340002, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774799756340002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:00.152755] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AB1-0-0] [lt=16] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1211, avg_group_log_size=121) [2024-08-16 10:20:00.152780] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AB1-0-0] [lt=23] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1211) [2024-08-16 10:20:00.153351] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1333) [2024-08-16 10:20:00.153676] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=27] current clock is (clock_=0, max_seq_=0, mem_limit=1643328294, cur_mem_hold=289406976, attr_.tenant_id_=1002) [2024-08-16 10:20:00.153705] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=22] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1723774783840797, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1723774800153600, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-08-16 10:20:00.153790] INFO [PALF] read (log_cache.cpp:100) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1333, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:00.159515] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=13] ====== tenant freeze timer task ====== [2024-08-16 10:20:00.161347] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [2774][T1001_LCLSender][T1001][Y0-0000000000000000-0-0] [lt=27] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=1135, total_destructed_detector=1135, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x7fa704be42b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-08-16 10:20:00.167704] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF077-0-0] [lt=9][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509853, log_task={header:{begin_lsn:{lsn:18544221230}, end_lsn:{lsn:18544221352}, log_id:53509853, min_scn:{val:1723774799878862001, v:0}, max_scn:{val:1723774799878862001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544221108}, prev_proposal_id:7, committed_end_lsn:{lsn:18544221230}, data_checksum:1913811101, accum_checksum:3623538191, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774800062641, freeze_ts:1723774800062641, submit_ts:1723774800062646, flushed_ts:1723774800063375, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:729}, fs_cb_begin_ts=1723774800167683, log_life_time=105042) [2024-08-16 10:20:00.176975] INFO [STORAGE] runTimerTask (ob_ddl_heart_beat_task.cpp:49) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=8] send to rs all task status succeed [2024-08-16 10:20:00.182260] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=36] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:00.182528] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash time detail, (compute_wash_size_time=114, refresh_score_time=67, wash_time=15) [2024-08-16 10:20:00.183437] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=20] gc stale ls task succ [2024-08-16 10:20:00.185883] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=14] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1572850, clean_num=62914) [2024-08-16 10:20:00.186951] INFO [CLOG.EXTLOG] resize_log_ext_handler_ (ob_cdc_service.cpp:632) [31748][T1_CdcSrv][T1][Y0-0000000000000000-0-0] [lt=21] finish to resize log external storage handler(current_ts=1723774800186948, tenant_max_cpu=2, valid_ls_count=0, other_ls_count=0, new_concurrency=0) [2024-08-16 10:20:00.196099] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=8] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:10008, add_timestamp:1723774800196062}) [2024-08-16 10:20:00.196135] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F0-0-0] [lt=43] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=10008, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.196177] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=22] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12358, add_timestamp:1723774800196173}) [2024-08-16 10:20:00.196218] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=23] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12396, add_timestamp:1723774800196216}) [2024-08-16 10:20:00.196624] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F0-0-0] [lt=17] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=10008, locations=[{table_id:10008, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.196685] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F0-0-0] [lt=59] success to process renew task(task={tenant_id:1, table_id:10008, add_timestamp:1723774800196062}, locations=[{table_id:10008, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}, {table_id:10008, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196594, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.196737] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F1-0-0] [lt=37] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12358, locations=[{table_id:12358, partition_id:12358, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196730, sql_renew_time:1723774800196730, is_mark_fail:false}]) [2024-08-16 10:20:00.196758] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F1-0-0] [lt=20] success to process renew task(task={tenant_id:1, table_id:12358, add_timestamp:1723774800196173}, locations=[{table_id:12358, partition_id:12358, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196730, sql_renew_time:1723774800196730, is_mark_fail:false}]) [2024-08-16 10:20:00.196781] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F2-0-0] [lt=12] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12396, locations=[{table_id:12396, partition_id:12396, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196776, sql_renew_time:1723774800196776, is_mark_fail:false}]) [2024-08-16 10:20:00.196798] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F2-0-0] [lt=16] success to process renew task(task={tenant_id:1, table_id:12396, add_timestamp:1723774800196216}, locations=[{table_id:12396, partition_id:12396, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800196776, sql_renew_time:1723774800196776, is_mark_fail:false}]) [2024-08-16 10:20:00.198701] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=13] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.198726] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=24] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.199819] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.199839] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:00.199849] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.199858] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.199871] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.200379] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=11] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.200426] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.200443] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=10] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.200450] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.200475] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=3] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.200488] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.200540] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:00.200556] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=15] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.200580] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=8] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.200585] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E00-0-0] [lt=5] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.230943] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=22] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=3) [2024-08-16 10:20:00.233011] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007E-00061EFF757F6EAD-0-0] [lt=20][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:00.236285] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=9] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:00.236381] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=12] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:00.236677] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:00.236903] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=17] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:00.237151] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=23] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:00.237262] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:00.237282] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=25] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:00.243410] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:00.243523] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:00.243539] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:00.243552] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:00.244275] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=650, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:00.244330] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=44] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=26, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:00.244350] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=14, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:00.244805] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5D8-0-0] [lt=71] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=15, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:00.244823] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5D8-0-0] [lt=16] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1260, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:00.245261] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:00.246030] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80086-00061EFF757FAF12-0-0] [lt=20][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:00.251055] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=25] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.251101] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=44] handle task succ(id={id:1260}) [2024-08-16 10:20:00.251113] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.251129] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.251145] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.262321] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE7F-0-0] [lt=24] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:00.262362] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE7F-0-0] [lt=24] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774800150054000, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774800150054000, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:00.262759] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF07D-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1614, avg_group_log_size=146) [2024-08-16 10:20:00.270726] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=18] current monitor number(seq_=-1) [2024-08-16 10:20:00.276173] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85EA-0-0] [lt=26] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1892) [2024-08-16 10:20:00.276793] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=597, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:00.276816] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1770) [2024-08-16 10:20:00.276839] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=33, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:00.276887] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=39] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=37, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:00.277226] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74399-0-0] [lt=28] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=14, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:00.277249] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74399-0-0] [lt=22] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=1086, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:00.295238] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=20] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574624, offset=42143377}, end_cursor:ObLogCursor{file_id=65, log_id=18574625, offset=42143618}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751680050, len:367, log_data_len:241, seq:18574624, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:00.295923] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=42] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574625, offset=42143618}, end_cursor:ObLogCursor{file_id=65, log_id=18574626, offset=42143867}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516ec050, len:4222, log_data_len:249, seq:18574625, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:00.297384] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA2-0-0] [lt=19] push gts task success(*task={:""}) [2024-08-16 10:20:00.299644] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=14] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:20:00.299664] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=18] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=5035, mem_used=39515136) [2024-08-16 10:20:00.299889] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.299909] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:00.299919] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.299929] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.299950] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.303411] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA2-0-0] [lt=18] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:00.304392] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=25] ====== tenant freeze timer task ====== [2024-08-16 10:20:00.304423] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=19][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:00.305809] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=20] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574626, offset=42143867}, end_cursor:ObLogCursor{file_id=65, log_id=18574627, offset=42144116}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75172c050, len:3973, log_data_len:249, seq:18574626, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:00.306230] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=21] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574627, offset=42144116}, end_cursor:ObLogCursor{file_id=65, log_id=18574628, offset=42144365}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751750050, len:3724, log_data_len:249, seq:18574627, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:00.337587] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E02-0-0] [lt=9] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11017, add_timestamp:1723774800337568}) [2024-08-16 10:20:00.337617] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F3-0-0] [lt=9] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11017, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.338034] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F3-0-0] [lt=17] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11017, locations=[{table_id:11017, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.338113] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F3-0-0] [lt=77] success to process renew task(task={tenant_id:1, table_id:11017, add_timestamp:1723774800337568}, locations=[{table_id:11017, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}, {table_id:11017, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800338010, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.342002] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_event.cpp:102) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E02-0-0] [lt=24][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:00.342626] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_event.cpp:102) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFADC-0-0] [lt=27][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:00.343489] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:00.343631] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:00.343646] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:00.343654] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:00.345342] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=25][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:00.348480] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFADD-0-0] [lt=19] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11003, add_timestamp:1723774800348447}) [2024-08-16 10:20:00.348506] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F4-0-0] [lt=36] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11003, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.349011] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F4-0-0] [lt=16] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11003, locations=[{table_id:11003, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.349068] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F4-0-0] [lt=56] success to process renew task(task={tenant_id:1, table_id:11003, add_timestamp:1723774800348447}, locations=[{table_id:11003, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}, {table_id:11003, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800348976, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.350249] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E03-0-0] [lt=22] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11095, add_timestamp:1723774800350233}) [2024-08-16 10:20:00.350268] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F5-0-0] [lt=82] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11095, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.350278] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=72] new mysql sessid created(conn.sessid_=3225602496, support_ssl=false) [2024-08-16 10:20:00.350329] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=15] sm conn init succ(conn.sessid_=3225602496, sess.client_addr_="127.0.0.1:51042") [2024-08-16 10:20:00.350345] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=11] accept one succ(*s={this:0x7fa733c7e030, session_id:3225602496, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:119}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:00.350398] INFO [SERVER] extract_user_tenant (obmp_connect.cpp:79) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=18] username and tenantname(user_name=ocp_monitor, tenant_name=) [2024-08-16 10:20:00.350448] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31556][sql_nio1][T1][Y0-0000000000000000-0-0] [lt=28] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:00.350456] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:280) [31556][sql_nio1][T1][Y0-0000000000000000-0-0] [lt=7] mysql login queue(mysql_queue->queue_.size()=1) [2024-08-16 10:20:00.350563] INFO load_privilege_info (obmp_connect.cpp:532) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24A-0-0] [lt=32] no tenant name set, use default tenant name(tenant_name=sys) [2024-08-16 10:20:00.350638] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F5-0-0] [lt=16] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11095, locations=[{table_id:11095, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.350686] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F5-0-0] [lt=47] success to process renew task(task={tenant_id:1, table_id:11095, add_timestamp:1723774800350233}, locations=[{table_id:11095, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}, {table_id:11095, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800350620, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.351044] WDIAG [COMMON] get (ob_link_hashmap.h:313) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24A-0-0] [lt=19][errcode=0] uref leak check(node->uref_=1074538681, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:00.351127] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.351146] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] handle task succ(id={id:1260}) [2024-08-16 10:20:00.351130] INFO [SERVER] process (obmp_connect.cpp:475) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24A-0-0] [lt=17] MySQL LOGIN(direct_client_ip="127.0.0.1", client_ip=127.0.0.1, tenant_name=sys, tenant_id=1, user_name=ocp_monitor, host_name=%, sessid=3225602496, proxy_sessid=0, sess_create_time=0, from_proxy=false, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=762509, proxy_capability=0, use_ssl=false, c/s protocol="OB_MYSQL_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:00.351155] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.351170] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.351184] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.351426] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E04-0-0] [lt=17] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11008, add_timestamp:1723774800351410}) [2024-08-16 10:20:00.351468] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F6-0-0] [lt=36] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11008, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.351922] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F6-0-0] [lt=20] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11008, locations=[{table_id:11008, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.351999] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F6-0-0] [lt=74] success to process renew task(task={tenant_id:1, table_id:11008, add_timestamp:1723774800351410}, locations=[{table_id:11008, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}, {table_id:11008, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800351907, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.353286] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa733c7e030, session_id:3225602496, trace_id:YB42C0A80080-00061CCCC65F2E05-0-0, sql_handling_stage:3, sql_initiative_shutdown:false, reader:{fd:119}, err:5, last_decode_time:1723774800352248, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:235, pending_flag:1, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:00.353312] WDIAG [SERVER] disconnect (obmp_packet_sender.cpp:830) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E05-0-0] [lt=31][errcode=0] server close connection(sessid=3225602496, proxy_sessid=0, stack="0x12a0688c 0xa219970 0xa1e504b 0xa527c01 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:00.353328] INFO [SERVER] process (obmp_quit.h:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E05-0-0] [lt=13] quit [2024-08-16 10:20:00.353326] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=33] kill and revert session(conn.sessid_=3225602496, proxy_sessid=0, server_id=16, ret=0) [2024-08-16 10:20:00.353338] WDIAG [RPC.OBMYSQL] async_write_data (ob_sql_sock_session.cpp:163) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E05-0-0] [lt=9][errcode=-4009] sock has error(ret=-4009) [2024-08-16 10:20:00.353347] WDIAG [SERVER] flush_buffer (obmp_packet_sender.cpp:1032) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E05-0-0] [lt=8][errcode=-4009] write response fail(ret=-4009) [2024-08-16 10:20:00.353377] WDIAG [SERVER] response (obmp_base.cpp:87) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E05-0-0] [lt=28][errcode=-4009] failed to flush_buffer(ret=-4009) [2024-08-16 10:20:00.353389] WDIAG [RPC.FRAME] run (ob_sql_processor.cpp:48) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E05-0-0] [lt=10][errcode=-4009] response rpc result fail(ret=-4009) [2024-08-16 10:20:00.362861] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF083-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1211) [2024-08-16 10:20:00.362914] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF083-0-0] [lt=32] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=15, avg_log_submit_to_flush_time=656, avg_log_submit_to_slide_time=100104) [2024-08-16 10:20:00.362934] INFO [PALF] read (log_cache.cpp:100) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=80] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:00.362930] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF083-0-0] [lt=14][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509855, log_task={header:{begin_lsn:{lsn:18544221465}, end_lsn:{lsn:18544221587}, log_id:53509855, min_scn:{val:1723774799878862003, v:0}, max_scn:{val:1723774799878862003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544221352}, prev_proposal_id:7, committed_end_lsn:{lsn:18544221465}, data_checksum:1913811101, accum_checksum:640676939, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774800262741, freeze_ts:1723774800262741, submit_ts:1723774800262746, flushed_ts:1723774800263412, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:666}, fs_cb_begin_ts=1723774800362898, log_life_time=100157) [2024-08-16 10:20:00.364807] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE0-0-0] [lt=20] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11021, add_timestamp:1723774800364792}) [2024-08-16 10:20:00.364833] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F7-0-0] [lt=95] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11021, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.365351] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F7-0-0] [lt=53] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11021, locations=[{table_id:11021, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.365423] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F7-0-0] [lt=70] success to process renew task(task={tenant_id:1, table_id:11021, add_timestamp:1723774800364792}, locations=[{table_id:11021, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800365337, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.369541] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE0-0-0] [lt=18][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:00.373121] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=17] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.373147] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=25] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.374916] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=16] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.374956] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.374970] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=10] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.374984] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=11] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.374996] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.375005] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.375058] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:00.375088] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=29] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.375114] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=15] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.375127] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE1-0-0] [lt=12] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.376229] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85EC-0-0] [lt=32] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1770, avg_group_log_size=160) [2024-08-16 10:20:00.384005] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=19] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:00.386364] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:00.386658] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=112, refresh_score_time=67, wash_time=19) [2024-08-16 10:20:00.389802] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1635764, clean_num=62914) [2024-08-16 10:20:00.399966] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.400002] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:00.400013] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.400022] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.400037] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.400030] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [2562][T1002_TabletShe][T1002][Y0-0000000000000000-0-0] [lt=11] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:20:00.400056] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:99) [2562][T1002_TabletShe][T1002][Y0-0000000000000000-0-0] [lt=15] [emptytablet] succeed to change tablet to empty shell(ret=0, ret="OB_SUCCESS", ls_cnt=4, times=12266) [2024-08-16 10:20:00.425208] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=21] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:00.425318] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=18] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:00.431052] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=28] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=2) [2024-08-16 10:20:00.435795] INFO [SQL.EXE] run2 (ob_maintain_dependency_info_task.cpp:221) [31493][MaintainDepInfo][T0][Y0-0000000000000000-0-0] [lt=20] [ASYNC TASK QUEUE](queue_size=0) [2024-08-16 10:20:00.443569] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:00.443682] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:00.443711] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:00.443723] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:00.445413] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:00.451210] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.451250] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=38] handle task succ(id={id:1260}) [2024-08-16 10:20:00.451263] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.451276] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.451294] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.451311] INFO [ARCHIVE] do_thread_task_ (ob_archive_fetcher.cpp:312) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] ObArchiveFetcher is running(thread_index=1) [2024-08-16 10:20:00.452489] INFO [STORAGE.TRANS] generate_new_version (ob_tenant_weak_read_server_version_mgr.cpp:120) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=34] [WRS] update tenant weak read server version(tenant_id=1001, server_version={version:{val:1723774795920970000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0, epoch_tstamp:1723774800452474}, version_delta=4531518) [2024-08-16 10:20:00.476335] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F738F51C-0-0] [lt=21][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509856, log_task={header:{begin_lsn:{lsn:18544221587}, end_lsn:{lsn:18544221709}, log_id:53509856, min_scn:{val:1723774799878862004, v:0}, max_scn:{val:1723774799878862004, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544221465}, prev_proposal_id:7, committed_end_lsn:{lsn:18544221587}, data_checksum:1913811101, accum_checksum:1771122923, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774800362841, freeze_ts:1723774800362841, submit_ts:1723774800362846, flushed_ts:1723774800363443, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:597}, fs_cb_begin_ts=1723774800476315, log_life_time=113474) [2024-08-16 10:20:00.477062] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [2767][T1001_LockWaitM][T1001][Y0-0000000000000000-0-0] [lt=22] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-08-16 10:20:00.489014] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE2-0-0] [lt=7] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12274, add_timestamp:1723774800488997}) [2024-08-16 10:20:00.489049] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F8-0-0] [lt=77] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=12274, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.489539] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F8-0-0] [lt=19] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12274, locations=[{table_id:12274, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.489599] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F8-0-0] [lt=59] success to process renew task(task={tenant_id:1, table_id:12274, add_timestamp:1723774800488997}, locations=[{table_id:12274, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}, {table_id:12274, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800489514, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.499999] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=14] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1002, server_version={version:{val:1723774799878862003, v:0}, total_part_count:4, valid_inner_part_count:0, valid_user_part_count:4}, server_version_delta=621133, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.124:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29408351, succ_cluster_heartbeat_count=29408341, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774715049399002, v:0}, local_cluster_delta=85450596, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:20:00.500055] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.500088] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=31] handle task succ(id={id:1260}) [2024-08-16 10:20:00.500102] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.500118] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.500145] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.525601] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=29] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=1, size_used=6922, mem_used=12478464) [2024-08-16 10:20:00.533053] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=34] start do ls ha handler(ls_id_array_=[{id:1260}, {id:1079}, {id:1261}, {id:1262}]) [2024-08-16 10:20:00.536137] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE6-0-0] [lt=38] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11018, add_timestamp:1723774800536119}) [2024-08-16 10:20:00.536168] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F9-0-0] [lt=44] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11018, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.536722] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F9-0-0] [lt=20] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11018, locations=[{table_id:11018, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.536807] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766F9-0-0] [lt=82] success to process renew task(task={tenant_id:1, table_id:11018, add_timestamp:1723774800536119}, locations=[{table_id:11018, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800536696, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.539124] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [31638][T1_Occam][T1][YB42C0A80080-00061CCCBF6D3E0A-0-0] [lt=19] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:00.540860] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=9] trace max work area(auto_calc=true, tenant_max_memory_limit=1073741824, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=53687091, max_wa_memory_size=53687091, tmp_max_wa_memory_size=53687091, pre_mem_target=53687091, remain_memory_size=53687091, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=332681216, washable_size=-2, max_workarea_memory_size=53687091, max_tenant_memory_size=741060608, wash_ratio=6, tenant_id=1001) [2024-08-16 10:20:00.540952] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=37] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=6710886, calc_global_bound_size=1048576000000, mem_target_=53687091, wa_max_memory_size=53687091, profile_cnt_=0, total_memory_size=0, max_wa_size=53687091, avg_bound_size=6710886, max_bound_size=6710886, min_bound_size_=1048576) [2024-08-16 10:20:00.540983] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=28] timer to calc global bound size(ret=0, global_bound_size_=6710886, manual_calc_cnt_=0, drift_size_=0, pre_drift_size=0, wa_max_memory_size=53687091, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1001, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=6710886, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=53687091, auto_calc=true, sql_mem_callback_.get_total_dump_size()=0) [2024-08-16 10:20:00.541032] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=13] trace max work area(auto_calc=true, tenant_max_memory_limit=6442450944, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=3865470566, max_wa_memory_size=3865470566, tmp_max_wa_memory_size=3865470566, pre_mem_target=3865470566, remain_memory_size=3865470566, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=2461601792, washable_size=-2, max_workarea_memory_size=3865470566, max_tenant_memory_size=3980849152, wash_ratio=6, tenant_id=1002) [2024-08-16 10:20:00.541114] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=21] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=483183820, calc_global_bound_size=1048576000000, mem_target_=3865470566, wa_max_memory_size=3865470566, profile_cnt_=0, total_memory_size=0, max_wa_size=3865470566, avg_bound_size=483183820, max_bound_size=483183820, min_bound_size_=1048576) [2024-08-16 10:20:00.541133] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=17] timer to calc global bound size(ret=0, global_bound_size_=483183820, manual_calc_cnt_=19637, drift_size_=0, pre_drift_size=0, wa_max_memory_size=3865470566, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1002, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=483183820, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=3865470566, auto_calc=true, sql_mem_callback_.get_total_dump_size()=1112786688) [2024-08-16 10:20:00.542523] WDIAG [SHARE] inner_get_ls_status_ (ob_ls_status_operator.cpp:955) [31743][T1_DupTbLease][T1][Y0-0000000000000000-0-0] [lt=21][errcode=-4018] ls not exist in __all_ls_status table(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-08-16 10:20:00.542592] WDIAG [SHARE] get_duplicate_ls_status_info (ob_ls_status_operator.cpp:674) [31743][T1_DupTbLease][T1][Y0-0000000000000000-0-0] [lt=20][errcode=-4018] fail to inner get ls status info(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sql=SELECT * FROM __all_ls_status where tenant_id = 1 and flag like "%DUPLICATE%", tenant_id=1, exec_tenant_id=1, need_member_list=false) [2024-08-16 10:20:00.543641] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:00.543741] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:00.543757] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:00.543764] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:00.545494] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:00.548422] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE7-0-0] [lt=25] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12034, add_timestamp:1723774800548393}) [2024-08-16 10:20:00.548453] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FA-0-0] [lt=41] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=12034, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.548889] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FA-0-0] [lt=32] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12034, locations=[{table_id:12034, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.548948] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FA-0-0] [lt=57] success to process renew task(task={tenant_id:1, table_id:12034, add_timestamp:1723774800548393}, locations=[{table_id:12034, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}, {table_id:12034, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800548867, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.551282] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.551302] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:00.551312] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.551321] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.551333] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.551652] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [31738][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=52] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:940, allocator used:3737440, allocator total:3859072}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:140, allocator used:9182740, allocator total:9182740}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=13041812, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:63, allocator used:124992, allocator total:130816}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:00.553002] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AB9-0-0] [lt=59] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:00.554780] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 352.00) [2024-08-16 10:20:00.554827] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E06-0-0] [lt=6] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12276, add_timestamp:1723774800554806}) [2024-08-16 10:20:00.554814] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=28] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:00.554862] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FB-0-0] [lt=69] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=12276, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.554898] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=31] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 5120.00, iops: 3.99, rt: 439.25) [2024-08-16 10:20:00.554913] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=11] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SHARED_BLOCK_ReaderWriter_IO, mode: read, size: 4096.00, iops: 5.98, rt: 306.33) [2024-08-16 10:20:00.554928] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:00.554969] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=32] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.554992] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=21] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555028] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=34] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555040] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=11] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555048] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555061] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=12] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555066] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555086] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=19] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555090] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555095] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555099] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555103] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555107] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555111] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555116] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555120] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:00.555302] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FB-0-0] [lt=27] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12276, locations=[{table_id:12276, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.555402] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FB-0-0] [lt=98] success to process renew task(task={tenant_id:1, table_id:12276, add_timestamp:1723774800554806}, locations=[{table_id:12276, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}, {table_id:12276, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800555276, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.569051] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=17] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12254, add_timestamp:1723774800569030}) [2024-08-16 10:20:00.569120] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FC-0-0] [lt=69] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=12254, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.569571] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FC-0-0] [lt=40] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12254, locations=[{table_id:12254, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.569658] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FC-0-0] [lt=85] success to process renew task(task={tenant_id:1, table_id:12254, add_timestamp:1723774800569030}, locations=[{table_id:12254, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}, {table_id:12254, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800569543, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.569730] WDIAG [SHARE] inner_get_ls_status_ (ob_ls_status_operator.cpp:955) [2757][T1001_DupTbLeas][T1001][Y0-0000000000000000-0-0] [lt=20][errcode=-4018] ls not exist in __all_ls_status table(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-08-16 10:20:00.569856] WDIAG [SHARE] get_duplicate_ls_status_info (ob_ls_status_operator.cpp:674) [2757][T1001_DupTbLeas][T1001][Y0-0000000000000000-0-0] [lt=18][errcode=-4018] fail to inner get ls status info(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sql=SELECT * FROM __all_ls_status where tenant_id = 1001 and flag like "%DUPLICATE%", tenant_id=1001, exec_tenant_id=1, need_member_list=false) [2024-08-16 10:20:00.570119] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=11] new mysql sessid created(conn.sessid_=3225600900, support_ssl=false) [2024-08-16 10:20:00.570182] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=26] sm conn init succ(conn.sessid_=3225600900, sess.client_addr_="127.0.0.1:51044") [2024-08-16 10:20:00.570209] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=20] accept one succ(*s={this:0x7fa70d9f4d70, session_id:3225600900, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:143}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:00.570255] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=31] can close safely, do destroy(*s={this:0x7fa733c7e030, session_id:3225602496, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:true, reader:{fd:119}, err:5, last_decode_time:1723774800352248, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:240, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-08-16 10:20:00.570308] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=20] unlock session of tenant(conn.sessid_=3225602496, proxy_sessid=0, conn.tenant_id_=1) [2024-08-16 10:20:00.570340] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21114][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=42][errcode=0] uref leak check(node->uref_=1074538681, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:00.570323] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=12] connection close(sessid=3225602496, proxy_sessid=0, tenant_id=1, server_id=16, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCC65F2E04-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:8, pkt_rec[0]:{send:obp_mysql_header_{len_:11, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{receive:obp_mysql_header_{len_:130, rec_:130, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[2]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{send:obp_mysql_header_{pkt_num_:3, seq_:5}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[4]:{send:obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_mysql_header_{pkt_num_:25, seq_:31}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[6]:{send:obp_mysql_header_{len_:9, seq_:32}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[7]:{receive:obp_mysql_header_{len_:1, rec_:1, seq_:0}, pkt_name:"COM_QUIT", obp_mysql_header_.type_:1, obp_mysql_header_.is_send_:0}], disconnect_state=3) [2024-08-16 10:20:00.570395] INFO [SERVER] extract_user_tenant (obmp_connect.cpp:79) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=47] username and tenantname(user_name=ocp_monitor, tenant_name=) [2024-08-16 10:20:00.570399] INFO [SERVER] run (obmp_disconnect.cpp:82) [21114][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=11] free session successfully(sessid=3225602496) [2024-08-16 10:20:00.570410] INFO [SERVER] run (obmp_disconnect.cpp:89) [21114][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=10] mark session id unused(sessid=3225602496) [2024-08-16 10:20:00.570432] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31556][sql_nio1][T1][Y0-0000000000000000-0-0] [lt=13] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:00.570537] INFO load_privilege_info (obmp_connect.cpp:532) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D2E-0-0] [lt=62] no tenant name set, use default tenant name(tenant_name=sys) [2024-08-16 10:20:00.570934] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D2E-0-0] [lt=16][errcode=0] uref leak check(node->uref_=1074538682, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:00.570996] INFO [SERVER] process (obmp_connect.cpp:475) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D2E-0-0] [lt=17] MySQL LOGIN(direct_client_ip="127.0.0.1", client_ip=127.0.0.1, tenant_name=sys, tenant_id=1, user_name=ocp_monitor, host_name=%, sessid=3225600900, proxy_sessid=0, sess_create_time=0, from_proxy=false, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=762509, proxy_capability=0, use_ssl=false, c/s protocol="OB_MYSQL_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:00.571314] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=69] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.571326] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21121][T1_L0_G0][T1][YB42C0A80080-00061CCCC201FAAD-0-0] [lt=27] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11037, add_timestamp:1723774800571311}) [2024-08-16 10:20:00.571334] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=10] new mysql sessid created(conn.sessid_=3225514112, support_ssl=false) [2024-08-16 10:20:00.571340] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=24] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.571344] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FD-0-0] [lt=92] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11037, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.571365] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=10] sm conn init succ(conn.sessid_=3225514112, sess.client_addr_="127.0.0.1:51046") [2024-08-16 10:20:00.571384] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=14] accept one succ(*s={this:0x7fa733c7e030, session_id:3225514112, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:119}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:00.571422] INFO [SERVER] extract_user_tenant (obmp_connect.cpp:79) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=12] username and tenantname(user_name=ocp_monitor, tenant_name=) [2024-08-16 10:20:00.571445] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31556][sql_nio1][T1][Y0-0000000000000000-0-0] [lt=7] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:00.571528] INFO load_privilege_info (obmp_connect.cpp:532) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24B-0-0] [lt=25] no tenant name set, use default tenant name(tenant_name=sys) [2024-08-16 10:20:00.571714] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FD-0-0] [lt=12] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11037, locations=[{table_id:11037, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.571766] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FD-0-0] [lt=51] success to process renew task(task={tenant_id:1, table_id:11037, add_timestamp:1723774800571311}, locations=[{table_id:11037, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}, {table_id:11037, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800571691, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.571912] WDIAG [COMMON] get (ob_link_hashmap.h:313) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24B-0-0] [lt=18][errcode=0] uref leak check(node->uref_=1074538682, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:00.571977] INFO [SERVER] process (obmp_connect.cpp:475) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24B-0-0] [lt=18] MySQL LOGIN(direct_client_ip="127.0.0.1", client_ip=127.0.0.1, tenant_name=sys, tenant_id=1, user_name=ocp_monitor, host_name=%, sessid=3225514112, proxy_sessid=0, sess_create_time=0, from_proxy=false, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=762509, proxy_capability=0, use_ssl=false, c/s protocol="OB_MYSQL_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:00.573055] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.573111] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=35] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.573124] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.573139] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=12] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.573149] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.573162] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=10] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.573193] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=5] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:00.573211] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=17] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.573237] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=12] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.573254] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E07-0-0] [lt=16] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.574158] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=5] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.574180] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.575991] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.576043] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.576066] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=18] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.576092] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=22] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.576101] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.576106] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.576150] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:00.576164] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=13] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.576182] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=10] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.576188] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56163-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.576730] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa733c7e030, session_id:3225514112, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:3, sql_initiative_shutdown:false, reader:{fd:119}, err:5, last_decode_time:1723774800576559, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:515, pending_flag:1, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:00.576767] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=29] kill and revert session(conn.sessid_=3225514112, proxy_sessid=0, server_id=16, ret=0) [2024-08-16 10:20:00.576773] WDIAG [SERVER] disconnect (obmp_packet_sender.cpp:830) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E09-0-0] [lt=11][errcode=0] server close connection(sessid=3225514112, proxy_sessid=0, stack="0x12a0688c 0xa219970 0xa1e504b 0xa527c01 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:00.576792] INFO [SERVER] process (obmp_quit.h:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E09-0-0] [lt=16] quit [2024-08-16 10:20:00.576802] WDIAG [RPC.OBMYSQL] async_write_data (ob_sql_sock_session.cpp:163) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E09-0-0] [lt=8][errcode=-4009] sock has error(ret=-4009) [2024-08-16 10:20:00.576809] WDIAG [SERVER] flush_buffer (obmp_packet_sender.cpp:1032) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E09-0-0] [lt=6][errcode=-4009] write response fail(ret=-4009) [2024-08-16 10:20:00.576813] WDIAG [SERVER] response (obmp_base.cpp:87) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E09-0-0] [lt=4][errcode=-4009] failed to flush_buffer(ret=-4009) [2024-08-16 10:20:00.576818] WDIAG [RPC.FRAME] run (ob_sql_processor.cpp:48) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E09-0-0] [lt=4][errcode=-4009] response rpc result fail(ret=-4009) [2024-08-16 10:20:00.577005] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF08D-0-0] [lt=23][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509857, log_task={header:{begin_lsn:{lsn:18544221709}, end_lsn:{lsn:18544222112}, log_id:53509857, min_scn:{val:1723774800379135000, v:0}, max_scn:{val:1723774800379135000, v:0}, data_len:347, proposal_id:7, prev_lsn:{lsn:18544221587}, prev_proposal_id:7, committed_end_lsn:{lsn:18544221709}, data_checksum:1991469776, accum_checksum:3565511881, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774800476293, freeze_ts:1723774800476293, submit_ts:1723774800476297, flushed_ts:1723774800477016, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:719}, fs_cb_begin_ts=1723774800576990, log_life_time=100697) [2024-08-16 10:20:00.577118] INFO [STORAGE.DUP_TABLE] follower_try_acquire_lease (ob_dup_table_lease.cpp:500) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=19] [DupLeaseLife] The follower can get new lease from this lease log(ret=0, ls_id={id:1079}, lease_log_scn={val:1723774800379135000, v:0}, self_addr="192.168.0.128:2882", acquire_new_lease=false, follower_lease_info_={durable_lease_:{request_ts_:1723774800144112, lease_interval_us_:6000000}, lease_expired_ts_:1723774806144112, last_lease_scn_:{val:1723774800379135000, v:0}, lease_acquire_scn_:{val:1722500589629865001, v:0}}) [2024-08-16 10:20:00.577572] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=21, stat_interval=1000000, avg (count/cost)=29, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:00.577889] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=12] can close safely, do destroy(*s={this:0x7fa733c7e030, session_id:3225514112, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:true, reader:{fd:119}, err:5, last_decode_time:1723774800576559, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:520, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-08-16 10:20:00.577933] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=31] unlock session of tenant(conn.sessid_=3225514112, proxy_sessid=0, conn.tenant_id_=1) [2024-08-16 10:20:00.577951] WDIAG [COMMON] get (ob_link_hashmap.h:313) [4774][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=17][errcode=0] uref leak check(node->uref_=1074538682, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:00.577987] INFO [SERVER] run (obmp_disconnect.cpp:82) [4774][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=13] free session successfully(sessid=3225514112) [2024-08-16 10:20:00.577955] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=20] connection close(sessid=3225514112, proxy_sessid=0, tenant_id=1, server_id=16, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCC1B56163-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:8, pkt_rec[0]:{send:obp_mysql_header_{len_:11, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{receive:obp_mysql_header_{len_:410, rec_:410, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[2]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{send:obp_mysql_header_{pkt_num_:3, seq_:5}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[4]:{send:obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_mysql_header_{pkt_num_:1, seq_:7}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[6]:{send:obp_mysql_header_{len_:9, seq_:8}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[7]:{receive:obp_mysql_header_{len_:1, rec_:1, seq_:0}, pkt_name:"COM_QUIT", obp_mysql_header_.type_:1, obp_mysql_header_.is_send_:0}], disconnect_state=3) [2024-08-16 10:20:00.578005] INFO [SERVER] run (obmp_disconnect.cpp:89) [4774][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=16] mark session id unused(sessid=3225514112) [2024-08-16 10:20:00.580058] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=16] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.580107] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=47] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.581666] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=13] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.581706] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.581715] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.581720] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.581726] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=3] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.581730] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=3] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.581761] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:00.581775] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=13] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.581792] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.581801] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120251-0-0] [lt=8] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.590296] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=812138496, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:00.590580] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash time detail, (compute_wash_size_time=117, refresh_score_time=66, wash_time=16) [2024-08-16 10:20:00.591119] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=9][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:00.593534] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=25] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:00.593626] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=28][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:00.593652] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=25][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:00.593661] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=9] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:00.594282] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=15] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=1698678, clean_num=62914) [2024-08-16 10:20:00.600108] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.600132] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:00.600149] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.600161] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.600181] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.600546] INFO [CLOG] runTimerTask (ob_log_replay_service.cpp:159) [31746][T1_ReplayProces][T1][Y0-0000000000000000-0-0] [lt=25] dump tenant replay process(tenant_id=1, unreplayed_log_size(MB)=0, estimate_time(second)=0, replayed_log_size(MB)=0, last_replayed_log_size(MB)=0, round_cost_time(second)=10, pending_replay_log_size(MB)=0) [2024-08-16 10:20:00.605435] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=3] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.605474] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=38] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.605969] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=5] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:14993, add_timestamp:1723774800605953}) [2024-08-16 10:20:00.605995] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FE-0-0] [lt=48] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=14993, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:00.606505] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FE-0-0] [lt=15] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=14993, locations=[{table_id:14993, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.606592] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FE-0-0] [lt=85] success to process renew task(task={tenant_id:1, table_id:14993, add_timestamp:1723774800605953}, locations=[{table_id:14993, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}, {table_id:14993, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774800606478, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:00.607007] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=18] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:00.607721] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.607730] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=22] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.607763] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.607773] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=8] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.607778] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.607785] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=3] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.607789] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=3] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.607817] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:00.607830] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=12] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.607851] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=10] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.607858] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.607961] INFO [PALF] get_stable_disk_usage (palf_env_impl.cpp:904) [31841][T1_L0_G0][T1001][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=6] get_stable_disk_usage(ret=0, capacity(MB):=2150, used(MB):=0) [2024-08-16 10:20:00.608005] INFO [PALF] get_stable_disk_usage (palf_env_impl.cpp:904) [31841][T1_L0_G0][T1002][YB42C0A80080-00061CCCC65F2E0D-0-0] [lt=18] get_stable_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:00.610002] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=20] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.610068] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=25] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.610110] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=34] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.610133] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=16] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.610169] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=10] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.610195] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=19] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.613976] WDIAG [SERVER] process_curr_tenant (ob_all_virtual_sys_stat.cpp:308) [21114][T1_L0_G0][T1001][YB42C0A80080-00061CCCC1B56164-0-0] [lt=11][errcode=-4008] get diag info fail(ret=-4008, tenant_id_=1001) [2024-08-16 10:20:00.614012] WDIAG [SERVER] process_curr_tenant (ob_all_virtual_sys_stat.cpp:308) [21114][T1_L0_G0][T1002][YB42C0A80080-00061CCCC1B56164-0-0] [lt=27][errcode=-4008] get diag info fail(ret=-4008, tenant_id_=1002) [2024-08-16 10:20:00.614022] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:00.614051] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:00.614121] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=58] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.614234] WDIAG [SERVER] process_curr_tenant (ob_all_virtual_sys_stat.cpp:308) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=17][errcode=-4008] get diag info fail(ret=-4008, tenant_id_=1) [2024-08-16 10:20:00.614258] WDIAG [SERVER] process_curr_tenant (ob_all_virtual_sys_stat.cpp:308) [21114][T1_L0_G0][T1001][YB42C0A80080-00061CCCC1B56164-0-0] [lt=19][errcode=-4008] get diag info fail(ret=-4008, tenant_id_=1001) [2024-08-16 10:20:00.615718] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=6][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:00.617158] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=35] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60674, mem_used=137263104) [2024-08-16 10:20:00.617168] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=11] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.617189] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=20] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.619060] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.619120] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=37] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.619149] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:00.619166] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.619180] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.619186] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=5] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.619217] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:00.619236] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=18] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.619260] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:00.619266] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E0E-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:00.621211] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56164-0-0] [lt=19][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:00.622910] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=22] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:00.631177] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=42] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=1) [2024-08-16 10:20:00.643722] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:00.643841] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=29] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:00.643860] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:00.643867] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:00.644951] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5E4-0-0] [lt=45] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=6, avg_log_gen_to_submit_time=64, avg_log_submit_to_flush_time=735, avg_log_submit_to_slide_time=1207) [2024-08-16 10:20:00.645531] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=24] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:20:00.645559] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=20] [tabletchange] task check ls(ls->get_ls_id()={id:1079}, tablet_persist_trigger=0) [2024-08-16 10:20:00.645571] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=9] [tabletchange] task check ls(ls->get_ls_id()={id:1260}, tablet_persist_trigger=0) [2024-08-16 10:20:00.645575] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [tabletchange] task check ls(ls->get_ls_id()={id:1261}, tablet_persist_trigger=0) [2024-08-16 10:20:00.645579] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [tabletchange] task check ls(ls->get_ls_id()={id:1262}, tablet_persist_trigger=0) [2024-08-16 10:20:00.645573] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=27][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:00.645584] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:228) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [tabletchange] succeed to gc_tablet(ret=0, ret="OB_SUCCESS", ls_cnt=4, times=11133) [2024-08-16 10:20:00.651365] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.651409] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=41] handle task succ(id={id:1260}) [2024-08-16 10:20:00.651426] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.651444] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.651476] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.653575] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=576, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:00.653655] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=32, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:00.653996] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA743A3-0-0] [lt=20] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=16, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:00.654016] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA743A3-0-0] [lt=19] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1051, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:00.700193] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.700237] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=41] handle task succ(id={id:1260}) [2024-08-16 10:20:00.700253] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.700265] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.700285] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.702267] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [31775][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=11] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:20:00.702323] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:101) [31775][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=47] [emptytablet] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12637) [2024-08-16 10:20:00.743787] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:00.743910] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=44, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:00.743951] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:00.743959] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:00.743967] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:00.743974] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:00.745497] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [2674][T1001_LogLoop][T1001][Y0-0000000000000000-0-0] [lt=20] LogLoopThread round_cost_time(us)(round_cost_time=2) [2024-08-16 10:20:00.745663] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:00.751408] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.751430] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:00.751441] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.751451] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.751466] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.776963] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF097-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:00.777049] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF097-0-0] [lt=52] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:00.777062] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF097-0-0] [lt=12] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=101508, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:00.777093] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF097-0-0] [lt=29][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509859, log_task={header:{begin_lsn:{lsn:18544222234}, end_lsn:{lsn:18544222356}, log_id:53509859, min_scn:{val:1723774800640543000, v:0}, max_scn:{val:1723774800640543000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544222112}, prev_proposal_id:7, committed_end_lsn:{lsn:18544222234}, data_checksum:1913811101, accum_checksum:595454521, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774800676963, freeze_ts:1723774800676963, submit_ts:1723774800676970, flushed_ts:1723774800677605, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:635}, fs_cb_begin_ts=1723774800777035, log_life_time=100072) [2024-08-16 10:20:00.777701] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1492) [2024-08-16 10:20:00.785286] WDIAG [SQL.ENG] inner_get_next_row (ob_px_dist_transmit_op.cpp:73) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF86ADC5E8-0-0] [lt=15][errcode=-4008] fail to exec ObPxTransmitOp::inner_get_next_row()(ret=-4008) [2024-08-16 10:20:00.787968] WDIAG [SQL.ENG] inner_get_next_row (ob_px_dist_transmit_op.cpp:73) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF86ADC5E8-0-0] [lt=26][errcode=-4008] fail to exec ObPxTransmitOp::inner_get_next_row()(ret=-4008) [2024-08-16 10:20:00.789131] WDIAG [SQL.ENG] inner_get_next_row (ob_px_dist_transmit_op.cpp:73) [31817][T1_L5_G0][T1][YB42C0A8007E-00061EFF89AB793F-0-0] [lt=21][errcode=-4008] fail to exec ObPxTransmitOp::inner_get_next_row()(ret=-4008) [2024-08-16 10:20:00.794775] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=24] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=816332800, global_cache_size=97796096, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:00.794901] INFO [COMMON] wash (ob_kvcache_store.cpp:335) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash memory, (tenant_id=1002, cache_size=54099968, lower_mem_limit=6442450944, upper_mem_limit=6442450944, min_wash_size=0, max_wash_size=0, mem_usage=2463698944, reserve_mem=355517662, wash_size=451588357) [2024-08-16 10:20:00.795001] WDIAG [SQL.ENG] inner_get_next_row (ob_px_dist_transmit_op.cpp:73) [31817][T1_L5_G0][T1][YB42C0A8007E-00061EFF89AB793F-0-0] [lt=22][errcode=-4008] fail to exec ObPxTransmitOp::inner_get_next_row()(ret=-4008) [2024-08-16 10:20:00.795096] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Wash time detail, (compute_wash_size_time=121, refresh_score_time=65, wash_time=51) [2024-08-16 10:20:00.798372] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=15] Cache wash clean map node details(ret=0, clean_node_count=17, clean_time=0, clean_start_pos=1761592, clean_num=62914) [2024-08-16 10:20:00.800269] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=33] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.800297] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:20:00.800278] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=46] get_max_applied_scn(scn={val:1723774800420970002, v:0}, this={ls_id_:{id:1260}, role_:1, proposal_id_:29, palf_committed_end_lsn_:{lsn:8828139561}, last_check_scn_:{val:1723774800420970003, v:0}, max_applied_cb_scn_:{val:1723774800420970002, v:0}}) [2024-08-16 10:20:00.800308] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.800317] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.800332] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.800313] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=29] get_min_unreplayed_log_info(lsn={lsn:8469424174}, scn={val:1723617929041268003, v:0}, this={ls_id_:{id:1260}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723617929662670, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:2710542, ret_code_:0}}, next_to_submit_lsn_:{lsn:8469424174}, next_to_submit_scn_:{val:1723617929041268003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516550107534003, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69fc05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8469424052}, end_lsn:{lsn:8469424174}, read_buf:{buf_len_:3682304, buf_:0x7fa69fc05000}, block_size:67104768, log_storage_:0x7fa6bd55b470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723617929041268002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723617929041268002, v:0}, data_checksum:3690972251, flag:0}}, init_mode_version:0, accumulate_checksum:2828504552, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723617929041268002, v:0}}}}}) [2024-08-16 10:20:00.800450] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=122] get_max_applied_scn(scn={val:1723617908966009002, v:0}, this={ls_id_:{id:1261}, role_:2, proposal_id_:4, palf_committed_end_lsn_:{lsn:32276646160}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1723617908966009002, v:0}}) [2024-08-16 10:20:00.800505] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=50] get_min_unreplayed_log_info(lsn={lsn:32601563446}, scn={val:1723774800362046005, v:0}, this={ls_id_:{id:1261}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774800777560, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:344950, ret_code_:0}}, next_to_submit_lsn_:{lsn:32601563446}, next_to_submit_scn_:{val:1723774800362046005, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516552080113001, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd805000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:32601563324}, end_lsn:{lsn:32601563446}, read_buf:{buf_len_:3682304, buf_:0x7fa6cd805000}, block_size:67104768, log_storage_:0x7fa6fe299470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774800362046004, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774800362046004, v:0}, data_checksum:1657047228, flag:1}}, init_mode_version:0, accumulate_checksum:649508666, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774800362046004, v:0}}}}}) [2024-08-16 10:20:00.800552] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=37] get_max_applied_scn(scn={val:0, v:0}, this={ls_id_:{id:1262}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:0, v:0}}) [2024-08-16 10:20:00.800564] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=9] get_min_unreplayed_log_info(lsn={lsn:8394844577}, scn={val:1723774800256809006, v:0}, this={ls_id_:{id:1262}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774800754290, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10679383, ret_code_:0}}, next_to_submit_lsn_:{lsn:8394844577}, next_to_submit_scn_:{val:1723774800256809006, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516551920929002, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69ec05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8394844455}, end_lsn:{lsn:8394844577}, read_buf:{buf_len_:3682304, buf_:0x7fa69ec05000}, block_size:67104768, log_storage_:0x7fa6be533470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774800256809005, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774800256809005, v:0}, data_checksum:2892311396, flag:1}}, init_mode_version:0, accumulate_checksum:755731062, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774800256809005, v:0}}}}}) [2024-08-16 10:20:00.805926] INFO [SERVER] init (ob_all_virtual_tenant_parameter_stat.cpp:64) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF788B4430-0-0] [lt=24] sys tenant show all local tenant parameter(effective_tenant_id_=1, tenant_id_list_=[1, 1001, 1002]) [2024-08-16 10:20:00.831287] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=18] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=0) [2024-08-16 10:20:00.843863] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:00.843969] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:00.843985] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:00.843992] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:00.844615] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:00.845734] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=28][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:00.851493] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.851514] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:00.851524] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.851534] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.851547] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.852726] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=30] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1001, server_version={version:{val:1723774795920970000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0}, server_version_delta=4931753, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.124:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29431336, succ_cluster_heartbeat_count=29431324, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774715706314000, v:0}, local_cluster_delta=85146409, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:20:00.876992] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF09B-0-0] [lt=41][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509860, log_task={header:{begin_lsn:{lsn:18544222356}, end_lsn:{lsn:18544222478}, log_id:53509860, min_scn:{val:1723774800640543001, v:0}, max_scn:{val:1723774800640543001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544222234}, prev_proposal_id:7, committed_end_lsn:{lsn:18544222356}, data_checksum:1913811101, accum_checksum:48229380, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774800776955, freeze_ts:1723774800776955, submit_ts:1723774800777018, flushed_ts:1723774800777776, gen_to_freeze cost time:0, gen_to_submit cost time:63, submit_to_flush cost time:758}, fs_cb_begin_ts=1723774800876979, log_life_time=100024) [2024-08-16 10:20:00.877610] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=595, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:00.900342] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.900366] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:00.900379] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.900391] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.900409] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.922707] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=58] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:00.932161] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [21119][T1_L0_G0][T1][YB42C0A80081-00061EFF86EC941A-0-0] [lt=20][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:00.939120] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=8] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1002) [2024-08-16 10:20:00.939168] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=35] start lib cache evict(tenant_id=1002, mem_hold=279150592, mem_limit=322122545, cache_obj_num=644, cache_node_num=575) [2024-08-16 10:20:00.939186] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=15] end lib cache evict(tenant_id=1002, cache_evict_num=0, mem_hold=279150592, mem_limit=322122545, cache_obj_num=644, cache_node_num=575) [2024-08-16 10:20:00.939200] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=9] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:00.942908] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=7] Dumping All Cache Objs(alloc_obj_list.count()=644, alloc_obj_list=[{obj_id:197058, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179184}, {obj_id:246944, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:166013}, {obj_id:246946, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175596}, {obj_id:199478, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205804}, {obj_id:249042, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:250289, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174834}, {obj_id:250293, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187797}, {obj_id:250294, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158389}, {obj_id:252064, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179845}, {obj_id:252089, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252117, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188403}, {obj_id:252210, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252318, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191338}, {obj_id:252323, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182962}, {obj_id:252333, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252334, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252336, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252340, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252341, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:252345, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221400}, {obj_id:252346, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284685}, {obj_id:252349, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181199}, {obj_id:252359, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:356776}, {obj_id:252372, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180356}, {obj_id:252377, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252386, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252404, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:382127}, {obj_id:252405, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252406, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:252409, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230382}, {obj_id:252417, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277038}, {obj_id:252425, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158366}, {obj_id:252428, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252440, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213028}, {obj_id:252445, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178214}, {obj_id:252447, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:252458, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:941831}, {obj_id:252459, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252464, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157801}, {obj_id:252475, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176302}, {obj_id:252485, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181511}, {obj_id:252488, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203010}, {obj_id:252489, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197827}, {obj_id:252495, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181747}, {obj_id:252502, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:705639}, {obj_id:252503, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:355214}, {obj_id:252511, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294158}, {obj_id:252513, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217004}, {obj_id:252540, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256131}, {obj_id:252541, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294112}, {obj_id:252554, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219188}, {obj_id:252569, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203562}, {obj_id:252573, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174535}, {obj_id:252586, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179957}, {obj_id:252587, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:254547}, {obj_id:252603, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294123}, {obj_id:252660, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196529}, {obj_id:252664, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294182}, {obj_id:252670, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292094}, {obj_id:252671, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294048}, {obj_id:252673, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211150}, {obj_id:252699, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202759}, {obj_id:252702, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252720, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157744}, {obj_id:252721, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:273791}, {obj_id:252730, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189375}, {obj_id:252749, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:399910}, {obj_id:252750, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:314035}, {obj_id:252764, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299083}, {obj_id:252765, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179021}, {obj_id:252769, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:355879}, {obj_id:252771, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188178}, {obj_id:252786, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237378}, {obj_id:252841, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177408}, {obj_id:252849, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252858, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180522}, {obj_id:252859, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180179}, {obj_id:252862, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6708112}, {obj_id:252864, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252865, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232017}, {obj_id:252868, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232266}, {obj_id:252869, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220840}, {obj_id:252939, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252955, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175994}, {obj_id:252962, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252964, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:349147}, {obj_id:252975, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252980, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252994, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252995, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252998, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253001, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253005, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253006, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253007, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253008, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294264}, {obj_id:253009, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292151}, {obj_id:253015, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298926}, {obj_id:253022, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301178}, {obj_id:253025, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298998}, {obj_id:253035, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298892}, {obj_id:253039, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:351399}, {obj_id:253040, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292087}, {obj_id:253047, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298992}, {obj_id:253050, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:288707}, {obj_id:253053, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204401}, {obj_id:253059, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208911}, {obj_id:253060, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211382}, {obj_id:253061, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200421}, {obj_id:253065, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223579}, {obj_id:253071, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290322}, {obj_id:253072, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250919}, {obj_id:253087, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253090, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:323458}, {obj_id:253091, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272373}, {obj_id:253094, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200393}, {obj_id:253095, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187302}, {obj_id:253096, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206572}, {obj_id:253097, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190787}, {obj_id:253098, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177031}, {obj_id:253102, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272582}, {obj_id:253107, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272526}, {obj_id:253113, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213908}, {obj_id:253114, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201513}, {obj_id:253115, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185332}, {obj_id:253122, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240210}, {obj_id:253124, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245608}, {obj_id:253127, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:372694}, {obj_id:253137, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1957477}, {obj_id:253169, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:270222}, {obj_id:253170, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272447}, {obj_id:253174, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:795070}, {obj_id:253234, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:283358}, {obj_id:253282, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253312, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286729}, {obj_id:253341, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253444, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157703}, {obj_id:253450, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186097}, {obj_id:253451, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:287939}, {obj_id:253452, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258575}, {obj_id:253454, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277108}, {obj_id:253457, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218998}, {obj_id:253458, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:285701}, {obj_id:253459, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253468, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181399}, {obj_id:253471, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:874259}, {obj_id:253473, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232858}, {obj_id:253476, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253477, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253478, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207070}, {obj_id:253480, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250296}, {obj_id:253485, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253486, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201045}, {obj_id:253487, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201415}, {obj_id:253488, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202980}, {obj_id:253489, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225769}, {obj_id:253490, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201728}, {obj_id:253491, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197763}, {obj_id:253497, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253500, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253506, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:11947420}, {obj_id:253507, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228298}, {obj_id:253508, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176153}, {obj_id:253509, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223501}, {obj_id:253510, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176542}, {obj_id:253511, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:253512, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218568}, {obj_id:253513, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177328}, {obj_id:253514, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177907}, {obj_id:253515, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184844}, {obj_id:253516, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253517, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177168}, {obj_id:253518, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:253519, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253520, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:253521, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:253522, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:253523, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253524, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253525, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253526, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253527, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:162334}, {obj_id:253529, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253530, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181547}, {obj_id:253531, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181340}, {obj_id:253532, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192983}, {obj_id:253533, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253534, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186898}, {obj_id:253535, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180995}, {obj_id:253536, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253537, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253538, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:253539, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253540, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223911}, {obj_id:253541, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253542, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221984}, {obj_id:253543, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186900}, {obj_id:253544, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213278}, {obj_id:253545, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6705734}, {obj_id:253546, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6739530}, {obj_id:253547, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202652}, {obj_id:253548, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253549, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:236673}, {obj_id:253550, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253551, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253553, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253554, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:367141}, {obj_id:253555, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189381}, {obj_id:253556, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253557, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253558, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205876}, {obj_id:253559, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253560, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189377}, {obj_id:253561, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253562, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237070}, {obj_id:253564, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205271}, {obj_id:253565, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238959}, {obj_id:253566, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292100}, {obj_id:253567, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:287824}, {obj_id:253568, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6741367}, {obj_id:253570, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187799}, {obj_id:253571, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253572, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175560}, {obj_id:253573, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175558}, {obj_id:253574, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:160948}, {obj_id:253576, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253578, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253579, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253580, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235030}, {obj_id:253581, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234962}, {obj_id:253582, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200857}, {obj_id:253583, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253584, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175142}, {obj_id:253585, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174672}, {obj_id:253588, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292043}, {obj_id:253589, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253590, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253591, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253594, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292025}, {obj_id:253595, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253596, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253601, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:383372}, {obj_id:253605, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292063}, {obj_id:253608, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290453}, {obj_id:253610, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253611, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:369179}, {obj_id:253614, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282531}, {obj_id:253615, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292282}, {obj_id:253616, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253617, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197701}, {obj_id:253618, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:476477}, {obj_id:253621, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297717}, {obj_id:253628, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:244896}, {obj_id:253630, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203464}, {obj_id:253631, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196891}, {obj_id:253632, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:253633, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186088}, {obj_id:253634, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181148}, {obj_id:253635, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253638, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:268654}, {obj_id:253640, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201594}, {obj_id:253641, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:265278}, {obj_id:253642, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290670}, {obj_id:253645, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253649, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292134}, {obj_id:253650, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206516}, {obj_id:253653, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290821}, {obj_id:253655, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238325}, {obj_id:253658, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198080}, {obj_id:253659, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216604}, {obj_id:253660, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178528}, {obj_id:253661, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:243178}, {obj_id:253664, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291977}, {obj_id:253665, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216113}, {obj_id:253666, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242569}, {obj_id:253667, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253668, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207760}, {obj_id:253669, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253672, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292096}, {obj_id:253677, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178509}, {obj_id:253681, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292081}, {obj_id:253682, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182898}, {obj_id:253683, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225569}, {obj_id:253691, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208341}, {obj_id:253692, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211962}, {obj_id:253694, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:379579}, {obj_id:253695, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185463}, {obj_id:253696, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253697, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253698, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194454}, {obj_id:253699, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185281}, {obj_id:253700, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277153}, {obj_id:253701, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196647}, {obj_id:253702, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220331}, {obj_id:253703, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253704, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228523}, {obj_id:253705, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184067}, {obj_id:253706, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175893}, {obj_id:253707, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189895}, {obj_id:253708, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251032}, {obj_id:253709, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189423}, {obj_id:253710, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:334817}, {obj_id:253711, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189681}, {obj_id:253712, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209440}, {obj_id:253713, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188174}, {obj_id:253714, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253715, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253716, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174832}, {obj_id:253723, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:932120}, {obj_id:253724, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:262469}, {obj_id:253725, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:332574}, {obj_id:253726, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240754}, {obj_id:253727, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210719}, {obj_id:253728, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253729, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:253730, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185730}, {obj_id:253731, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250192}, {obj_id:253732, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177398}, {obj_id:253733, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191786}, {obj_id:253734, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177488}, {obj_id:253735, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276369}, {obj_id:253736, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219016}, {obj_id:253737, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:510255}, {obj_id:253740, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197323}, {obj_id:253743, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234930}, {obj_id:253747, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210144}, {obj_id:253749, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253750, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253751, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227821}, {obj_id:253753, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:302184}, {obj_id:253754, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253755, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183280}, {obj_id:253756, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253757, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282524}, {obj_id:253758, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200791}, {obj_id:253759, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:384669}, {obj_id:253760, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:382379}, {obj_id:253761, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:873919}, {obj_id:253762, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:870651}, {obj_id:253763, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:516499}, {obj_id:253764, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253765, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253766, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204396}, {obj_id:253767, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222427}, {obj_id:253768, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253769, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221512}, {obj_id:253770, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:554188}, {obj_id:253771, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247245}, {obj_id:253772, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253773, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196589}, {obj_id:253774, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187792}, {obj_id:253775, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198057}, {obj_id:253776, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279996}, {obj_id:253777, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221914}, {obj_id:253779, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180817}, {obj_id:253780, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179884}, {obj_id:253782, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225027}, {obj_id:253784, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250315}, {obj_id:253791, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292016}, {obj_id:253792, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291532}, {obj_id:253793, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180590}, {obj_id:253796, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250098}, {obj_id:253797, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180228}, {obj_id:253798, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176856}, {obj_id:253799, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195067}, {obj_id:253800, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253801, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177751}, {obj_id:253802, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208961}, {obj_id:253805, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240180}, {obj_id:253808, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256449}, {obj_id:253809, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:449159}, {obj_id:253810, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188388}, {obj_id:253811, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199543}, {obj_id:253812, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195231}, {obj_id:253813, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253814, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253815, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214158}, {obj_id:253816, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253817, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:288727}, {obj_id:253820, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291994}, {obj_id:253821, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235062}, {obj_id:253827, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175823}, {obj_id:253828, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174483}, {obj_id:253829, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217784}, {obj_id:253830, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177814}, {obj_id:253831, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:444256}, {obj_id:253832, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195622}, {obj_id:253833, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253834, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:497014}, {obj_id:253835, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253836, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:261672}, {obj_id:253837, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:435156}, {obj_id:253838, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:516719}, {obj_id:253839, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200167}, {obj_id:253840, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271961}, {obj_id:253841, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194596}, {obj_id:253842, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205257}, {obj_id:253843, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174397}, {obj_id:253845, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253846, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253847, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253848, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224798}, {obj_id:253849, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223872}, {obj_id:253850, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224553}, {obj_id:253851, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221103}, {obj_id:253852, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219500}, {obj_id:253853, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219425}, {obj_id:253854, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253855, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240470}, {obj_id:253856, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253857, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253858, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220267}, {obj_id:253859, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220212}, {obj_id:253861, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290476}, {obj_id:253864, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:547504}, {obj_id:253865, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233724}, {obj_id:253866, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218340}, {obj_id:253867, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197559}, {obj_id:253868, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253869, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217547}, {obj_id:253870, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235957}, {obj_id:253871, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237233}, {obj_id:253872, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:280008}, {obj_id:253873, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208674}, {obj_id:253874, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190144}, {obj_id:253875, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240637}, {obj_id:253876, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187604}, {obj_id:253878, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241654}, {obj_id:253879, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253880, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253881, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176921}, {obj_id:253882, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188072}, {obj_id:253883, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:246455}, {obj_id:253884, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176040}, {obj_id:253886, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253887, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253890, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, adde [2024-08-16 10:20:00.943746] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=834] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:00.943754] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [31787][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=18] tx gc loop thread is running(MTL_ID()=1) [2024-08-16 10:20:00.943946] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:00.944054] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:00.944066] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:00.944089] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:00.945811] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:00.951606] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:00.951627] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:00.951640] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:00.951683] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=40] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:00.951712] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] try consume task status succ(id={id:1260}) [2024-08-16 10:20:00.951730] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=20] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:00.951853] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=114] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:00.952008] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:00.952044] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=16][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:00.952057] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:00.952068] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=9][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:00.954583] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F735BA-0-0] [lt=47] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=9, avg_log_submit_to_flush_time=636, avg_log_submit_to_slide_time=1055) [2024-08-16 10:20:00.955979] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2417][T1002_Occam][T1002][YB42C0A80080-00061CCCBF7D3860-0-0] [lt=25] for each ls(succ_num=4, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:00.975406] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [31728][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=23] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:20:00.977939] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA743AD-0-0] [lt=36] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=11, avg_log_submit_to_flush_time=703, avg_log_submit_to_slide_time=1157) [2024-08-16 10:20:00.977975] INFO [PALF] read (log_cache.cpp:100) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=36] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1667, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:00.998888] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=812138496, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:00.999231] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=47] Wash time detail, (compute_wash_size_time=163, refresh_score_time=92, wash_time=24) [2024-08-16 10:20:01.000445] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.000490] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=41] handle task succ(id={id:1260}) [2024-08-16 10:20:01.000523] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=30] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.000534] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.000555] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.002842] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=12, clean_time=0, clean_start_pos=1824506, clean_num=62914) [2024-08-16 10:20:01.012267] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=35] gc stale ls task succ [2024-08-16 10:20:01.012372] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=3] gc stale ls task succ [2024-08-16 10:20:01.012408] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=21] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:01.012446] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=26] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:01.012479] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=30] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:01.017016] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:01.017111] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:01.025848] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C12-0-0] [lt=34] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774801025810) [2024-08-16 10:20:01.033443] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=26] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=2642388, replace_num=31457) [2024-08-16 10:20:01.033492] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=45] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=10) [2024-08-16 10:20:01.044036] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:01.044265] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=65] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=11, avg_log_batch_cnt=1, total_group_log_size=1342, avg_group_log_size=122) [2024-08-16 10:20:01.044305] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=34] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1342) [2024-08-16 10:20:01.044357] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=47] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:01.044370] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:01.044379] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:01.044837] INFO [STORAGE.DUP_TABLE] ls_loop_handle (ob_dup_table_util.cpp:624) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=26] [DUP_TABLE_DIAG: Dup Interface Stat] tenant: 1002, ls: 1079, is_master: False, {dup_table_follower_read_succ_cnt_:780757, dup_table_follower_read_tablet_not_exist_cnt_:0, dup_table_follower_read_tablet_not_ready_cnt_:0, dup_table_follower_read_lease_expired_cnt_:71, dup_table_redo_sync_succ_cnt_:0, dup_table_redo_sync_fail_cnt_:0, dup_table_max_applying_scn_:{val:1723774800379135000, v:0}, dup_table_log_entry_cnt_:2761310, dup_table_log_entry_total_size_:1070043114, dup_table_log_replay_total_time_:0, dup_table_log_deser_total_time_:0, dup_table_lease_log_sync_total_time_:4042973, dup_table_tablet_log_sync_total_time_:2501514} [2024-08-16 10:20:01.044890] INFO [STORAGE.DUP_TABLE] print_lease_diag_info_log (ob_dup_table_lease.cpp:658) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=23] [DUP_TABLE_DIAG: Follower Lease Info] tenant: 1002, ls: 1079 , current_time = 2024-08-16 10:20:01.44861 [DUP_TABLE_DIAG: Follower Lease] is_expired=False, request_ts=1723774800144112, request_ts(date)=2024-08-16 10:20:00.144112, lease_expired_time=2024-08-16 10:20:06.144112, lease_interval_us=6000000, last_lease_scn={val:1723774800379135000, v:0}, lease_acquire_scn={val:1722500589629865001, v:0} [2024-08-16 10:20:01.045890] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:01.051688] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.051714] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:20:01.051761] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=41] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.051776] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.051800] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.052989] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=40] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:01.064541] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1723774801064537, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007632, add_bytes: 0 [2024-08-16 10:20:01.064978] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:01.065015] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=35][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:01.076799] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=23] PNIO [ratelimit] time: 1723774801076796, bytes: 1496436610096, bw: 0.252674 MB/s, add_ts: 1000902, add_bytes: 265187 [2024-08-16 10:20:01.076881] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85FA-0-0] [lt=35] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:01.077333] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0A1-0-0] [lt=30][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509861, log_task={header:{begin_lsn:{lsn:18544222478}, end_lsn:{lsn:18544222600}, log_id:53509861, min_scn:{val:1723774800814781000, v:0}, max_scn:{val:1723774800814781000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544222356}, prev_proposal_id:7, committed_end_lsn:{lsn:18544222478}, data_checksum:1913811101, accum_checksum:3046915227, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774800876962, freeze_ts:1723774800876962, submit_ts:1723774800876965, flushed_ts:1723774800877670, gen_to_freeze cost time:0, gen_to_submit cost time:3, submit_to_flush cost time:705}, fs_cb_begin_ts=1723774801077314, log_life_time=200352) [2024-08-16 10:20:01.077481] INFO [LIB] stat (utility.h:1197) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=91] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=29, stat_interval=1000000, avg (count/cost)=41, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:01.077514] INFO [LIB] stat (utility.h:1197) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=31] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=29, stat_interval=1000000, avg (count/cost)=16, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:01.078313] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=29] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=32, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:20:01.100526] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.100558] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=29] handle task succ(id={id:1260}) [2024-08-16 10:20:01.100579] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.100592] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.100616] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.126197] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2659][T1001_Occam][T1001][YB42C0A80080-00061CCCBF8D3860-0-0] [lt=21] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:01.127374] INFO [PALF] runTimerTask (block_gc_timer_task.cpp:101) [2672][T1001_PalfGC][T1001][Y0-0000000000000000-0-0] [lt=29] BlockGCTimerTask success(ret=0, cost_time_us=8, palf_env_impl_={IPalfEnvImpl:{IPalfEnvImpl:"Dummy"}, self:"192.168.0.128:2882", log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1001", disk_options_wrapper:{disk_opts_for_stopping_writing:{log_disk_size(MB):2150, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):2150, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}, log_alloc_mgr_:{flying_log_task:0, flying_meta_task:0}}) [2024-08-16 10:20:01.129372] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DB-0-0] [lt=98][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:01.130109] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DB-0-0] [lt=21] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:01.138844] WDIAG [SQL.ENG] inner_get_next_row (ob_px_dist_transmit_op.cpp:73) [21121][T1_L0_G0][T1][YB42C0A8007C-000616D3EE3DAEF4-0-0] [lt=32][errcode=-4008] fail to exec ObPxTransmitOp::inner_get_next_row()(ret=-4008) [2024-08-16 10:20:01.144130] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:01.144260] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:01.144275] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:01.144283] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:01.144848] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:01.145968] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:120) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23] tenant info is invalid(ret=-4076, query_ts=0, standby_scn={val:0, v:0}, this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:20:01.146004] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=34][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:01.146477] INFO [STORAGE.TRANS] run1 (ob_xa_trans_heartbeat_worker.cpp:83) [2553][T1002_ObXAHbWor][T1002][Y0-0000000000000000-0-0] [lt=27] XA scheduler heartbeat task statistics(avg_time=18) [2024-08-16 10:20:01.151723] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.151746] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:01.151757] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.151769] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.151785] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.153566] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AC5-0-0] [lt=72] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:01.153631] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AC5-0-0] [lt=60] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:01.154120] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:01.154710] INFO [PALF] read (log_cache.cpp:100) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=41] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:01.162364] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007F-000616D421E91A0D-0-0] [lt=21][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:01.164224] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AAE-0-0] [lt=20] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:01.164280] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AAE-0-0] [lt=24] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774800757535003, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774800757535003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:01.180022] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E10-0-0] [lt=14] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11031, add_timestamp:1723774801180000}) [2024-08-16 10:20:01.180053] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FF-0-0] [lt=66] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11031, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:01.180440] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E10-0-0] [lt=35][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=375, pos=320, bitmap=NULL) [2024-08-16 10:20:01.180531] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1044) [31728][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=26] success to update min reserved snapshot(reserved_snapshot=1723772992385636000, duration=1800, snapshot_gc_ts_=1723774792385636000) [2024-08-16 10:20:01.180566] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1071) [31728][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=27] update reserved snapshot finished(cost_ts=38, reserved_snapshot=1723772992385636000) [2024-08-16 10:20:01.180659] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FF-0-0] [lt=41] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11031, locations=[{table_id:11031, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:01.180719] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC96766FF-0-0] [lt=59] success to process renew task(task={tenant_id:1, table_id:11031, add_timestamp:1723774801180000}, locations=[{table_id:11031, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774801180634, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:01.180821] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E10-0-0] [lt=42][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=20, len=241, pos=211, bitmap=NULL) [2024-08-16 10:20:01.181128] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E10-0-0] [lt=20][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=869, pos=385, bitmap=NULL) [2024-08-16 10:20:01.183540] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=17] gc stale ls task succ [2024-08-16 10:20:01.187153] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCC998AE0A8-0-0] [lt=13] receive register_req from acceptor, ignore(self="192.168.0.128:2882", palf_id=1260, child={server:"192.168.0.125:2882", region:"sys_region", register_time_us:1723774801185991, keepalive_ts:-1}) [2024-08-16 10:20:01.187188] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCC998AE0A8-0-0] [lt=33] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1260, self="192.168.0.128:2882", child={server:"192.168.0.125:2882", region:"sys_region", register_time_us:1723774801185991, keepalive_ts:-1}, is_to_leader=true) [2024-08-16 10:20:01.200607] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.200636] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:20:01.200649] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.200659] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.200674] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.203510] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=814235648, global_cache_size=97796096, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:01.203718] INFO [COMMON] wash (ob_kvcache_store.cpp:335) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=36] Wash memory, (tenant_id=1002, cache_size=54099968, lower_mem_limit=6442450944, upper_mem_limit=6442450944, min_wash_size=0, max_wash_size=0, mem_usage=2463698944, reserve_mem=355517662, wash_size=450428230) [2024-08-16 10:20:01.203971] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash time detail, (compute_wash_size_time=151, refresh_score_time=82, wash_time=110) [2024-08-16 10:20:01.207535] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=37] Cache wash clean map node details(ret=0, clean_node_count=35, clean_time=0, clean_start_pos=1887420, clean_num=62914) [2024-08-16 10:20:01.221847] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [31738][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=79] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:940, allocator used:3737440, allocator total:3859072}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:140, allocator used:9182740, allocator total:9182740}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=13041812, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:63, allocator used:124992, allocator total:130816}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:01.233597] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=13] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=9) [2024-08-16 10:20:01.236119] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=23] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:01.236246] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:01.236418] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:01.236606] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:01.236656] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=17] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:01.237179] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:01.237577] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:01.242129] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD16DAE254-0-0] [lt=17] receive register_req from acceptor, ignore(self="192.168.0.128:2882", palf_id=1260, child={server:"192.168.0.133:2882", region:"sys_region", register_time_us:1723774801239805, keepalive_ts:-1}) [2024-08-16 10:20:01.242170] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD16DAE254-0-0] [lt=38] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1260, self="192.168.0.128:2882", child={server:"192.168.0.133:2882", region:"sys_region", register_time_us:1723774801239805, keepalive_ts:-1}, is_to_leader=true) [2024-08-16 10:20:01.244223] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:01.244381] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=39] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:01.244399] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:01.244406] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:01.245109] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=616, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:01.245157] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=31] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=34, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:01.245184] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=29] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=17, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:01.245655] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5F5-0-0] [lt=19] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:01.245681] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5F5-0-0] [lt=25] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1335, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:01.246043] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=10][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:01.246823] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [31653][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=21] LogLoopThread round_cost_time(us)(round_cost_time=2) [2024-08-16 10:20:01.251815] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.251847] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=29] handle task succ(id={id:1260}) [2024-08-16 10:20:01.251862] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.251875] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.251921] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=34] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.270846] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=19] current monitor number(seq_=-1) [2024-08-16 10:20:01.276961] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD85FE-0-0] [lt=10] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:01.277509] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=661, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:01.277538] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:01.277563] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=31, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:01.277614] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=42, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:01.278111] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5F6-0-0] [lt=26] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:01.278131] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE5F6-0-0] [lt=19] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1234, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:01.278552] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0AD-0-0] [lt=22] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=9, avg_log_batch_cnt=0, total_group_log_size=1379, avg_group_log_size=153) [2024-08-16 10:20:01.278588] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0AD-0-0] [lt=21][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509863, log_task={header:{begin_lsn:{lsn:18544222722}, end_lsn:{lsn:18544222844}, log_id:53509863, min_scn:{val:1723774800880378001, v:0}, max_scn:{val:1723774800880378001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544222600}, prev_proposal_id:7, committed_end_lsn:{lsn:18544222722}, data_checksum:1913811101, accum_checksum:3664308829, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774801177302, freeze_ts:1723774801177302, submit_ts:1723774801177306, flushed_ts:1723774801177840, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:534}, fs_cb_begin_ts=1723774801278577, log_life_time=101275) [2024-08-16 10:20:01.280924] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE80-0-0] [lt=23] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:01.280963] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE80-0-0] [lt=22] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774800862145004, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774800862145004, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:01.296993] INFO [PALF] get_stable_disk_usage (palf_env_impl.cpp:904) [21121][T1_L0_G0][T1001][YB42C0A8007E-00061EFF855F81C3-0-0] [lt=35] get_stable_disk_usage(ret=0, capacity(MB):=2150, used(MB):=0) [2024-08-16 10:20:01.297044] INFO [PALF] get_stable_disk_usage (palf_env_impl.cpp:904) [21121][T1_L0_G0][T1002][YB42C0A8007E-00061EFF855F81C3-0-0] [lt=25] get_stable_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:01.297143] WDIAG [SQL.ENG] inner_get_next_row (ob_px_dist_transmit_op.cpp:73) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF855F81C3-0-0] [lt=6][errcode=-4008] fail to exec ObPxTransmitOp::inner_get_next_row()(ret=-4008) [2024-08-16 10:20:01.299802] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=13] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:20:01.299826] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=22] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=5100, mem_used=39515136) [2024-08-16 10:20:01.300661] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.300688] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:20:01.300699] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.300710] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.300726] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.307927] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=20] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574628, offset=42144365}, end_cursor:ObLogCursor{file_id=65, log_id=18574629, offset=42144606}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75165c050, len:3475, log_data_len:241, seq:18574628, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:01.308741] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=56] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574629, offset=42144606}, end_cursor:ObLogCursor{file_id=65, log_id=18574630, offset=42144855}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516f4050, len:3234, log_data_len:249, seq:18574629, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:01.310446] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA3-0-0] [lt=18] push gts task success(*task={:""}) [2024-08-16 10:20:01.318141] INFO [SQL.PC] need_check_schema_version (ob_plan_cache_value.cpp:2169) [2512][T1002_FreInfoRe][T1001][YB42C0A80080-00061CCCC7BE5A3C-0-0] [lt=85] need check schema(new_schema_version=1722932441982352, cached_tenant_schema_version=1722932441982352, is_contain_synonym()=false, contain_sys_name_table_=true, is_contain_tmp_tbl()=false, is_contain_sys_pl_object()=false, pc_ctx.sql_ctx_.session_info_->get_has_temp_table_flag()=false, need_check=true, constructed_sql_=SELECT * FROM __all_merge_info WHERE tenant_id = ?) [2024-08-16 10:20:01.319846] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA3-0-0] [lt=17] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:01.322668] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=30] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574630, offset=42144855}, end_cursor:ObLogCursor{file_id=65, log_id=18574631, offset=42145104}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751670050, len:2985, log_data_len:249, seq:18574630, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:01.323366] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=31] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574631, offset=42145104}, end_cursor:ObLogCursor{file_id=65, log_id=18574632, offset=42145353}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751658050, len:2736, log_data_len:249, seq:18574631, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:01.324190] INFO [CLOG] run1 (ob_garbage_collector.cpp:1338) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=10] Garbage Collector is running(seq_=325421, gc_interval=10000000) [2024-08-16 10:20:01.324247] INFO [CLOG] construct_server_ls_map_for_member_list_ (ob_garbage_collector.cpp:1473) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=20] self is leader, skip it(ls->get_ls_id()={id:1260}) [2024-08-16 10:20:01.325534] INFO [CLOG] handle_rpc_response_ (ob_garbage_collector.cpp:272) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=19] GC check ls in member list, skip it(id={id:1079}, leader="192.168.0.124:2882") [2024-08-16 10:20:01.326110] INFO [CLOG] handle_rpc_response_ (ob_garbage_collector.cpp:272) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=20] GC check ls in member list, skip it(id={id:1261}, leader="192.168.0.125:2882") [2024-08-16 10:20:01.326337] INFO [COMMON] process (ob_rpc_processor_simple.cpp:2405) [13775][T1002_L0_G19][T1002][YB42C0A80085-00061CCD15ECDA3B-0-0] [lt=27] handle ObQueryLSIsValidMember requeset(arg_={tenant_id_:1002, self_addr_:"192.168.0.133:2882", ls_array_:[{id:1260}]}) [2024-08-16 10:20:01.326377] INFO [CLOG] get_member_gc_stat (ob_log_handler.cpp:1509) [13775][T1002_L0_G19][T1002][YB42C0A80085-00061CCD15ECDA3B-0-0] [lt=27] get_member_gc_stat(is_valid_member=true, stat=1, member_list=3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, learner_list={learner_num:0, learner_array:[]}, addr="192.168.0.133:2882", this={role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}) [2024-08-16 10:20:01.326577] INFO [CLOG] handle_rpc_response_ (ob_garbage_collector.cpp:272) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=24] GC check ls in member list, skip it(id={id:1262}, leader="192.168.0.133:2882") [2024-08-16 10:20:01.326610] INFO [CLOG] gc_check_member_list_ (ob_garbage_collector.cpp:1431) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=30] gc_check_member_list_ cost time(ret=0, time_us=2399) [2024-08-16 10:20:01.326628] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1703) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=9] execute_gc cost time(ret=0, time_us=5) [2024-08-16 10:20:01.328812] INFO [CLOG] gc_check_ls_status_ (ob_garbage_collector.cpp:1626) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=8] gc_candidates push_back success(ret=0, candidate={ls_id_:{id:1079}, ls_status_:1, gc_reason_:0}) [2024-08-16 10:20:01.330593] INFO [CLOG] gc_check_ls_status_ (ob_garbage_collector.cpp:1626) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=21] gc_candidates push_back success(ret=0, candidate={ls_id_:{id:1260}, ls_status_:1, gc_reason_:0}) [2024-08-16 10:20:01.331450] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=32] get all snapshots(ret=0, snapshots.count()=192, snapshots=[{snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213191, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213192, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213193, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213194, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213195, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213196, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213197, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213198, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213199, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213200, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213201, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213202, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213203, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213204, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213205, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213206, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213207, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213208, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213209, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213210, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213211, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213212, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213213, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213214, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213215, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213216, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213217, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213218, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213219, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213220, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213221, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213222, comment:null}]) [2024-08-16 10:20:01.332511] INFO [CLOG] gc_check_ls_status_ (ob_garbage_collector.cpp:1626) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=22] gc_candidates push_back success(ret=0, candidate={ls_id_:{id:1261}, ls_status_:1, gc_reason_:0}) [2024-08-16 10:20:01.334494] INFO [CLOG] gc_check_ls_status_ (ob_garbage_collector.cpp:1626) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=21] gc_candidates push_back success(ret=0, candidate={ls_id_:{id:1262}, ls_status_:1, gc_reason_:0}) [2024-08-16 10:20:01.334553] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1648) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=50] ls status is normal, skip(id={id:1079}, gc_candidates=[{ls_id_:{id:1079}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1260}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1261}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1262}, ls_status_:1, gc_reason_:0}]) [2024-08-16 10:20:01.334567] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1648) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=14] ls status is normal, skip(id={id:1260}, gc_candidates=[{ls_id_:{id:1079}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1260}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1261}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1262}, ls_status_:1, gc_reason_:0}]) [2024-08-16 10:20:01.334575] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1648) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=8] ls status is normal, skip(id={id:1261}, gc_candidates=[{ls_id_:{id:1079}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1260}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1261}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1262}, ls_status_:1, gc_reason_:0}]) [2024-08-16 10:20:01.334602] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1648) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=26] ls status is normal, skip(id={id:1262}, gc_candidates=[{ls_id_:{id:1079}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1260}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1261}, ls_status_:1, gc_reason_:0}, {ls_id_:{id:1262}, ls_status_:1, gc_reason_:0}]) [2024-08-16 10:20:01.334612] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1703) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=9] execute_gc cost time(ret=0, time_us=61) [2024-08-16 10:20:01.334620] INFO [SERVER] handle (ob_safe_destroy_handler.cpp:240) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=6] ObSafeDestroyHandler start process [2024-08-16 10:20:01.334628] INFO [SERVER] loop (ob_safe_destroy_handler.cpp:133) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=6] ObSafeDestroyTaskQueue::loop begin(queue_.size()=0) [2024-08-16 10:20:01.334653] INFO [SERVER] loop (ob_safe_destroy_handler.cpp:140) [2534][T1002_GCCollect][T1002][Y0-0000000000000000-0-0] [lt=10] ObSafeDestroyTaskQueue::loop finish(ret=0, queue_.size()=0) [2024-08-16 10:20:01.344374] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:01.344537] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=30] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:01.344557] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:01.344574] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:01.346129] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:01.351899] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.351924] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:20:01.351936] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.351947] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.351962] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.370577] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [2798][T1001_TxLoopWor][T1001][Y0-0000000000000000-0-0] [lt=8] tx gc loop thread is running(MTL_ID()=1001) [2024-08-16 10:20:01.370602] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [2798][T1001_TxLoopWor][T1001][Y0-0000000000000000-0-0] [lt=23] try gc retain ctx [2024-08-16 10:20:01.377025] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8600-0-0] [lt=33] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:01.378966] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0B3-0-0] [lt=31] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1379) [2024-08-16 10:20:01.379004] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0B3-0-0] [lt=23] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=11, avg_log_submit_to_flush_time=692, avg_log_submit_to_slide_time=112925) [2024-08-16 10:20:01.379029] INFO [PALF] read (log_cache.cpp:100) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=36] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1379, hit_cnt=9, read_cnt=9, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:01.380963] INFO [CLOG] run1 (ob_garbage_collector.cpp:1338) [31752][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=12] Garbage Collector is running(seq_=326001, gc_interval=10000000) [2024-08-16 10:20:01.381004] INFO [CLOG] gc_check_member_list_ (ob_garbage_collector.cpp:1431) [31752][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=20] gc_check_member_list_ cost time(ret=0, time_us=20) [2024-08-16 10:20:01.381021] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1703) [31752][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=12] execute_gc cost time(ret=0, time_us=1) [2024-08-16 10:20:01.381033] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1703) [31752][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=8] execute_gc cost time(ret=0, time_us=0) [2024-08-16 10:20:01.381041] INFO [SERVER] handle (ob_safe_destroy_handler.cpp:240) [31752][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=5] ObSafeDestroyHandler start process [2024-08-16 10:20:01.384118] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=25] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:01.386975] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1961) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(dag_cnt=0, map_size=0) [2024-08-16 10:20:01.387009] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1971) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=33] dump_dag_status(running_dag_net_map_size=0, blocking_dag_net_list_size=0) [2024-08-16 10:20:01.387017] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(priority="PRIO_COMPACTION_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:01.387033] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=15] dump_dag_status(priority="PRIO_HA_HIGH", low_limit=8, up_limit=8, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:01.387037] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_COMPACTION_MID", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:01.387041] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_HA_MID", low_limit=5, up_limit=5, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:01.387045] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_COMPACTION_LOW", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:01.387049] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_HA_LOW", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:01.387053] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_DDL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:01.387057] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_DDL_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:01.387061] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_TTL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:01.387065] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:01.387116] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=49] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:01.387128] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=11] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:01.387135] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:01.387141] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:01.387145] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:01.387148] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:01.387152] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:01.387156] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:01.387164] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:01.387168] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:01.387173] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:01.387179] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:01.387193] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=13] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:01.387198] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:01.387202] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:01.387205] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:01.387209] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:01.387215] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:01.387225] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:01.387230] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387234] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387238] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387243] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387248] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387253] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387256] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387260] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387264] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387268] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387271] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387275] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387279] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387283] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387287] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387291] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387294] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387298] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387302] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387306] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387309] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387313] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387317] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387321] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387324] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387328] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387332] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387336] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387339] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:01.387345] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387349] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-08-16 10:20:01.387353] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-08-16 10:20:01.387357] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-08-16 10:20:01.387368] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=11] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-08-16 10:20:01.387373] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:01.387379] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:01.387389] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:01.387394] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:01.387398] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:01.387403] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:01.387414] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:01.387419] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:01.387423] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:01.387427] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:01.387430] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:01.387434] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:01.387438] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:01.387443] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:01.387454] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:01.387459] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:01.387463] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:01.387466] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:01.387470] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:01.387474] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387477] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:01.387481] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387485] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:01.387489] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387493] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:01.387497] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387500] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:01.387506] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387516] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:01.387534] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=17] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387544] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:01.387552] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387562] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:01.387593] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=31] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387601] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:01.387607] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387615] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:01.387629] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=13] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:01.387638] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, dag_count=0) [2024-08-16 10:20:01.387644] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, scheduled_task_count=0) [2024-08-16 10:20:01.387653] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, dag_count=0) [2024-08-16 10:20:01.387672] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=18] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, scheduled_task_count=0) [2024-08-16 10:20:01.387682] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, dag_count=0) [2024-08-16 10:20:01.387695] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=13] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-08-16 10:20:01.387700] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, dag_count=0) [2024-08-16 10:20:01.387705] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-08-16 10:20:01.387709] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, dag_count=0) [2024-08-16 10:20:01.387713] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, scheduled_task_count=0) [2024-08-16 10:20:01.387716] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_MIGRATION", dag_net_count=0) [2024-08-16 10:20:01.387721] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status[DAG_NET](type="DAG_NET_PREPARE_MIGRATION", dag_net_count=0) [2024-08-16 10:20:01.387724] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_COMPLETE_MIGRATION", dag_net_count=0) [2024-08-16 10:20:01.387727] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER", dag_net_count=0) [2024-08-16 10:20:01.387730] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_BACKUP", dag_net_count=0) [2024-08-16 10:20:01.387733] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_RESTORE", dag_net_count=0) [2024-08-16 10:20:01.387736] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TYPE_BACKUP_CLEAN", dag_net_count=0) [2024-08-16 10:20:01.387740] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER_BACKFILL_TX", dag_net_count=0) [2024-08-16 10:20:01.387744] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1991) [31727][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(total_worker_cnt=43, total_running_task_cnt=0, work_thread_num=43, scheduled_task_cnt=0) [2024-08-16 10:20:01.400737] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.400763] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:20:01.400776] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.400787] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.400799] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.408202] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=24] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=814235648, global_cache_size=97796096, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:01.408347] INFO [COMMON] wash (ob_kvcache_store.cpp:335) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=49] Wash memory, (tenant_id=1002, cache_size=54099968, lower_mem_limit=6442450944, upper_mem_limit=6442450944, min_wash_size=0, max_wash_size=0, mem_usage=2463698944, reserve_mem=355517662, wash_size=450428230) [2024-08-16 10:20:01.408568] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash time detail, (compute_wash_size_time=186, refresh_score_time=70, wash_time=47) [2024-08-16 10:20:01.411647] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=15] Cache wash clean map node details(ret=0, clean_node_count=46, clean_time=0, clean_start_pos=1950334, clean_num=62914) [2024-08-16 10:20:01.423322] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=34] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1, need_refresh=false, gts_local_cache={srr:[mts=1723774800922660], gts:1723774800923331000, latest_srr:[mts=1723774801423287]}) [2024-08-16 10:20:01.423366] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=32] gts statistics(tenant_id=1001, gts_rpc_cnt=22, get_gts_cache_cnt=107, get_gts_with_stc_cnt=12, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-08-16 10:20:01.423381] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=14] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_refresh=false, gts_local_cache={srr:[mts=1723774801314301], gts:1723774801313378000, latest_srr:[mts=1723774801423355]}) [2024-08-16 10:20:01.423567] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=9] gts statistics(tenant_id=1002, gts_rpc_cnt=11, get_gts_cache_cnt=371, get_gts_with_stc_cnt=1, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=1, try_wait_gts_elapse_cnt=0) [2024-08-16 10:20:01.423582] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=14] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_refresh=false, gts_local_cache={srr:[mts=1723774800922923], gts:1723774800921508000, latest_srr:[mts=1723774801423546]}) [2024-08-16 10:20:01.424575] INFO [CLOG.EXTLOG] run1 (ob_cdc_service.cpp:172) [2530][T1002_CdcSrv][T1002][Y0-0000000000000000-0-0] [lt=20] query dest_info_ succ(dest_info=[[0, "oss://ep-ob-test-back/ep_test_cluster/ep_test_cluster/1713182723/tenant_incarnation_1/1002/clog?host=oss-cn-hangzhou-internal.aliyuncs.com&access_id=LTAI5tFcMQ9HN1VQNfbd72o6&access_key=9yNDBp9XhZUnNJMXRMP3MgZQt7rPch"]], archive_dest_changed=false) [2024-08-16 10:20:01.424616] INFO [CLOG.EXTLOG] resize_log_ext_handler_ (ob_cdc_service.cpp:632) [2530][T1002_CdcSrv][T1002][Y0-0000000000000000-0-0] [lt=24] finish to resize log external storage handler(current_ts=1723774801424612, tenant_max_cpu=8, valid_ls_count=0, other_ls_count=0, new_concurrency=0) [2024-08-16 10:20:01.425293] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=37] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:01.425450] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=21] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:01.427637] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=9] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1) [2024-08-16 10:20:01.427667] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=23] start lib cache evict(tenant_id=1, mem_hold=46137344, mem_limit=53687090, cache_obj_num=123, cache_node_num=123) [2024-08-16 10:20:01.427681] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=12] end lib cache evict(tenant_id=1, cache_evict_num=0, mem_hold=46137344, mem_limit=53687090, cache_obj_num=123, cache_node_num=123) [2024-08-16 10:20:01.427692] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=9] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:01.429655] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [31771][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=7] ====== checkpoint timer task ====== [2024-08-16 10:20:01.429688] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [31771][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=25] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:20:01.430692] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=6] Dumping All Cache Objs(alloc_obj_list.count()=123, alloc_obj_list=[{obj_id:27, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190650}, {obj_id:95, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:96, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:174, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:984, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205451}, {obj_id:1112, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:310633}, {obj_id:1476, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179375}, {obj_id:1489, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191695}, {obj_id:1687, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198485}, {obj_id:1732, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172160}, {obj_id:1736, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174033}, {obj_id:1741, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175220}, {obj_id:1758, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205820}, {obj_id:1862, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188466}, {obj_id:2017, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:2151, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189565}, {obj_id:2278, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:2279, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189832}, {obj_id:2280, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176478}, {obj_id:2281, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177168}, {obj_id:2282, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174044}, {obj_id:2283, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174081}, {obj_id:2284, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174036}, {obj_id:2285, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185923}, {obj_id:2286, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220611}, {obj_id:2287, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207401}, {obj_id:2288, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180621}, {obj_id:2289, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177481}, {obj_id:2290, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:2291, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183781}, {obj_id:2292, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2293, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188441}, {obj_id:2294, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260630}, {obj_id:2295, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223446}, {obj_id:2296, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173811}, {obj_id:2297, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172864}, {obj_id:2298, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179482}, {obj_id:2299, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186746}, {obj_id:2300, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204032}, {obj_id:2301, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188714}, {obj_id:2302, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183948}, {obj_id:2303, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182890}, {obj_id:2304, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2305, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:2306, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2307, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:2308, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2309, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:2310, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2311, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:2312, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203568}, {obj_id:2313, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191582}, {obj_id:2314, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182638}, {obj_id:2315, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182415}, {obj_id:2316, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184713}, {obj_id:2317, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182209}, {obj_id:2318, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205928}, {obj_id:2319, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223722}, {obj_id:2320, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238225}, {obj_id:2321, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194888}, {obj_id:2322, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220593}, {obj_id:2323, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188416}, {obj_id:2325, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188394}, {obj_id:2327, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196424}, {obj_id:2330, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229007}, {obj_id:2333, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194173}, {obj_id:2336, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210443}, {obj_id:2337, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206506}, {obj_id:2338, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180446}, {obj_id:2339, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234757}, {obj_id:2340, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179743}, {obj_id:2341, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179601}, {obj_id:2342, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184831}, {obj_id:2343, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188563}, {obj_id:2344, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237176}, {obj_id:2345, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198348}, {obj_id:2346, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256258}, {obj_id:2347, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185263}, {obj_id:2348, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196309}, {obj_id:2349, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:265571}, {obj_id:2350, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260809}, {obj_id:2351, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256181}, {obj_id:2352, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299476}, {obj_id:2353, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:270176}, {obj_id:2354, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:305943}, {obj_id:2355, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172528}, {obj_id:2356, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176862}, {obj_id:2357, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183129}, {obj_id:2358, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175603}, {obj_id:2359, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181253}, {obj_id:2360, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181039}, {obj_id:2361, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185755}, {obj_id:2362, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202273}, {obj_id:2363, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:267576}, {obj_id:2364, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183894}, {obj_id:2365, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217508}, {obj_id:2366, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:3, added_to_lc:true, mem_used:211579}, {obj_id:2367, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184855}, {obj_id:2368, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228602}, {obj_id:2369, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217173}, {obj_id:2370, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214453}, {obj_id:2371, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206378}, {obj_id:2372, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178773}, {obj_id:2373, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177464}, {obj_id:2374, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230030}, {obj_id:2375, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177376}, {obj_id:2376, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183047}, {obj_id:2377, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210108}, {obj_id:2378, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202314}, {obj_id:2379, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202343}, {obj_id:2380, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198396}, {obj_id:2381, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229073}, {obj_id:2382, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204285}, {obj_id:2383, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179141}, {obj_id:2384, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2385, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179385}, {obj_id:2386, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180644}, {obj_id:2387, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2388, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175472}, {obj_id:2389, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175472}, {obj_id:2390, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2391, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174540}, {obj_id:2392, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188792}]) [2024-08-16 10:20:01.430904] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=207] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:01.433712] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=34] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=8) [2024-08-16 10:20:01.444446] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:01.444586] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=50] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:01.444600] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:01.444606] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:01.446214] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=34][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:01.449716] INFO [STORAGE.TRANS] refresh_elr_tenant_config_ (ob_tx_elr_util.cpp:47) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E094E-0-0] [lt=17] refresh tenant config success(tenant_id=1002, *this={last_refresh_ts:1723774801449425, can_tenant_elr:true}) [2024-08-16 10:20:01.450896] INFO [STORAGE.TRANS] alloc (ob_trans_factory.cpp:141) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E094F-0-0] [lt=32] ObTransCtx statistics(active_coord_ctx_count=0, active_part_ctx_count=1, total_release_part_ctx_count=2) [2024-08-16 10:20:01.450977] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E094F-0-0] [lt=21] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1723774791444304, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1723774801450499, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-08-16 10:20:01.451120] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E094F-0-0] [lt=23] current clock is (clock_=0, max_seq_=0, mem_limit=1643328294, cur_mem_hold=289406976, attr_.tenant_id_=1002) [2024-08-16 10:20:01.451982] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.452004] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:01.452016] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.452029] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.452051] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.454343] WDIAG [SERVER] do_process (obmp_query.cpp:1018) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E0951-0-0] [lt=21][errcode=0] need disconnect(ret=0, need_disconnect=true) [2024-08-16 10:20:01.479529] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0B9-0-0] [lt=12][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509865, log_task={header:{begin_lsn:{lsn:18544222966}, end_lsn:{lsn:18544223088}, log_id:53509865, min_scn:{val:1723774800880378003, v:0}, max_scn:{val:1723774800880378003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544222844}, prev_proposal_id:7, committed_end_lsn:{lsn:18544222966}, data_checksum:1913811101, accum_checksum:4141930853, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774801378947, freeze_ts:1723774801378947, submit_ts:1723774801378952, flushed_ts:1723774801379520, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:568}, fs_cb_begin_ts=1723774801479517, log_life_time=100570) [2024-08-16 10:20:01.488273] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7DE-0-0] [lt=22] [RS_MGR] force set rs list(master_rs="192.168.0.126:2882") [2024-08-16 10:20:01.488329] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7DE-0-0] [lt=43] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"192.168.0.128:2882", server_id:16, rs_addr:"192.168.0.126:2882", rs_server_status:1, epoch_id:11}, hb_response={zone:"zone2", server:"192.168.0.128:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1720514727760575, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=57) [2024-08-16 10:20:01.500846] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.500877] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=28] handle task succ(id={id:1260}) [2024-08-16 10:20:01.500889] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.500900] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.500915] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.518649] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [31651][T1_PalfGC][T1][Y0-0000000000000000-0-0] [lt=51] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}) [2024-08-16 10:20:01.525753] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=39] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=6936, mem_used=12478464) [2024-08-16 10:20:01.533220] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=44] start do ls ha handler(ls_id_array_=[{id:1261}, {id:1260}, {id:1262}, {id:1079}]) [2024-08-16 10:20:01.544200] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [31787][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=16] try gc retain ctx [2024-08-16 10:20:01.544522] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:01.544553] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:01.544562] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:01.544569] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:01.546308] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=24][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:01.552066] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=38] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.552112] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=43] handle task succ(id={id:1260}) [2024-08-16 10:20:01.552125] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.552137] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.552152] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.553820] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849ACD-0-0] [lt=49] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:01.556859] INFO [SHARE] refresh_primary_cluster_id (ob_alive_server_tracer.cpp:397) [31391][ServerTracerTim][T0][YB42C0A80080-00061CCC8EABC3A2-0-0] [lt=25] refresh primary cluster finish(ret=0, primary_cluster_id_=-1) [2024-08-16 10:20:01.558254] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 319.00) [2024-08-16 10:20:01.558293] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=30] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:01.558388] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=50] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.99, rt: 572.00) [2024-08-16 10:20:01.558415] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=19] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SHARED_BLOCK_ReaderWriter_IO, mode: read, size: 4185.37, iops: 2420.53, rt: 228.72) [2024-08-16 10:20:01.558424] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: ROOT_BLOCK_IO, mode: read, size: 8192.00, iops: 1.00, rt: 269.00) [2024-08-16 10:20:01.558437] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=3, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:01.558464] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=21] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558530] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=2, req_count=1, reservation_ts=1723774801558477, group_limitation_ts=1723774801558477, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558550] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=18] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558560] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558570] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558606] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=35] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558615] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558621] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558629] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558636] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558643] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558649] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558657] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558665] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558680] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=14] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.558689] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:01.564148] INFO [SERVER] init (ob_all_virtual_tenant_parameter_stat.cpp:64) [21121][T1_L0_G0][T1][YB42C0A8007C-000616D3EE3DAEF8-0-0] [lt=31] sys tenant show all local tenant parameter(effective_tenant_id_=1, tenant_id_list_=[1, 1001, 1002]) [2024-08-16 10:20:01.568684] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [31733][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=27] run rewrite rule refresh task(rule_mgr_->tenant_id_=1) [2024-08-16 10:20:01.579666] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0BF-0-0] [lt=35][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509866, log_task={header:{begin_lsn:{lsn:18544223088}, end_lsn:{lsn:18544223210}, log_id:53509866, min_scn:{val:1723774801380864000, v:0}, max_scn:{val:1723774801380864000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544222966}, prev_proposal_id:7, committed_end_lsn:{lsn:18544223088}, data_checksum:1913811101, accum_checksum:164128978, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774801479498, freeze_ts:1723774801479499, submit_ts:1723774801479503, flushed_ts:1723774801480110, gen_to_freeze cost time:1, gen_to_submit cost time:5, submit_to_flush cost time:607}, fs_cb_begin_ts=1723774801579650, log_life_time=100152) [2024-08-16 10:20:01.580233] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=39] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=19, stat_interval=1000000, avg (count/cost)=39, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:01.584499] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=7] ====== check clog disk timer task ====== [2024-08-16 10:20:01.584529] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=26] get_disk_usage(ret=0, capacity(MB):=2150, used(MB):=0) [2024-08-16 10:20:01.584551] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=14] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=676457349, need_update_checkpoint_scn=false) [2024-08-16 10:20:01.585577] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=8] ====== check clog disk timer task ====== [2024-08-16 10:20:01.585605] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=25] get_disk_usage(ret=0, capacity(MB):=0, used(MB):=0) [2024-08-16 10:20:01.585646] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=21] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=0, need_update_checkpoint_scn=false) [2024-08-16 10:20:01.591199] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=19][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:01.593454] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=40] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:01.593548] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=27][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:01.593571] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=21][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:01.593578] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=6] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:01.600925] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.600952] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:20:01.600965] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.600979] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.600998] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.607152] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=60] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:01.612150] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=814235648, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:01.612429] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash time detail, (compute_wash_size_time=155, refresh_score_time=66, wash_time=16) [2024-08-16 10:20:01.615654] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=40, clean_time=0, clean_start_pos=2013248, clean_num=62914) [2024-08-16 10:20:01.617347] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=55] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=1, size_used=60684, mem_used=137263104) [2024-08-16 10:20:01.627488] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=20] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:01.633835] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=23] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=7) [2024-08-16 10:20:01.644613] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:01.644736] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:01.644752] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:01.644759] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:01.645463] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [31772][T1_Flush][T1][Y0-0000000000000000-0-0] [lt=7] ====== traversal_flush timer task ====== [2024-08-16 10:20:01.645491] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:338) [31772][T1_Flush][T1][Y0-0000000000000000-0-0] [lt=22] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:20:01.645969] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F735CD-0-0] [lt=74] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=5, avg_log_gen_to_submit_time=75, avg_log_submit_to_flush_time=703, avg_log_submit_to_slide_time=1262) [2024-08-16 10:20:01.646385] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=50][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:01.647248] INFO [SERVER] init (ob_all_virtual_tenant_parameter_stat.cpp:64) [21121][T1_L0_G0][T1][YB42C0A8007C-000616D3EE3DAEFD-0-0] [lt=30] sys tenant show all local tenant parameter(effective_tenant_id_=1, tenant_id_list_=[1, 1001, 1002]) [2024-08-16 10:20:01.652136] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.652157] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:01.652168] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.652179] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.652194] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.654501] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=620, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:01.654599] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=39, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:01.655132] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA743C0-0-0] [lt=31] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=14, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:01.655159] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA743C0-0-0] [lt=25] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1161, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:01.700996] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.701023] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:20:01.701035] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.701046] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.701061] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.735987] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=10] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1001) [2024-08-16 10:20:01.736021] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=29] start lib cache evict(tenant_id=1001, mem_hold=48234496, mem_limit=53687090, cache_obj_num=115, cache_node_num=115) [2024-08-16 10:20:01.736036] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=12] end lib cache evict(tenant_id=1001, cache_evict_num=0, mem_hold=48234496, mem_limit=53687090, cache_obj_num=115, cache_node_num=115) [2024-08-16 10:20:01.736056] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=16] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:01.739263] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=8] Dumping All Cache Objs(alloc_obj_list.count()=115, alloc_obj_list=[{obj_id:4206, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190714}, {obj_id:6967, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:7888, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190463}, {obj_id:8067, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198580}, {obj_id:8069, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179360}, {obj_id:8462, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:252321}, {obj_id:8631, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176478}, {obj_id:8633, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:8641, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188033}, {obj_id:8645, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8646, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:8647, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8648, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:8649, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8651, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8652, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8653, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188613}, {obj_id:8682, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187574}, {obj_id:8692, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234101}, {obj_id:8697, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227212}, {obj_id:8767, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245433}, {obj_id:8786, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279819}, {obj_id:8795, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:281759}, {obj_id:8805, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191548}, {obj_id:8840, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:293038}, {obj_id:8842, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:324921}, {obj_id:8843, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:456542}, {obj_id:8851, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:575647}, {obj_id:8854, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:347535}, {obj_id:8873, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:452546}, {obj_id:8878, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:582753}, {obj_id:8885, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:474728}, {obj_id:8886, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:329638}, {obj_id:8891, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:452691}, {obj_id:8892, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:375165}, {obj_id:8896, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:356761}, {obj_id:8897, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290927}, {obj_id:8898, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:413784}, {obj_id:8906, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8907, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8908, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189807}, {obj_id:8909, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189568}, {obj_id:8910, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176333}, {obj_id:8912, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196245}, {obj_id:8913, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8914, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180998}, {obj_id:8915, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174064}, {obj_id:8916, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177216}, {obj_id:8917, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:8918, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196176}, {obj_id:8919, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177512}, {obj_id:8920, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8921, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:8922, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:8923, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180749}, {obj_id:8925, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:253523}, {obj_id:8926, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199084}, {obj_id:8927, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178773}, {obj_id:8928, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177464}, {obj_id:8929, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179416}, {obj_id:8930, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180672}, {obj_id:8931, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8932, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186657}, {obj_id:8933, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233944}, {obj_id:8934, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185434}, {obj_id:8935, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198144}, {obj_id:8936, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207413}, {obj_id:8937, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189999}, {obj_id:8938, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188514}, {obj_id:8939, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188880}, {obj_id:8940, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231829}, {obj_id:8941, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190949}, {obj_id:8942, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192036}, {obj_id:8943, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187766}, {obj_id:8944, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195808}, {obj_id:8946, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176485}, {obj_id:8947, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:386890}, {obj_id:8948, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8949, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202973}, {obj_id:8950, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183752}, {obj_id:8951, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223501}, {obj_id:8952, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188501}, {obj_id:8953, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186993}, {obj_id:8954, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:8955, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:8956, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:8957, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176676}, {obj_id:8958, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175405}, {obj_id:8959, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199144}, {obj_id:8960, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205144}, {obj_id:8961, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218139}, {obj_id:8962, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218827}, {obj_id:8963, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217152}, {obj_id:8964, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218674}, {obj_id:8965, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8966, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206801}, {obj_id:8967, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256033}, {obj_id:8968, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8969, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199798}, {obj_id:8970, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199390}, {obj_id:8971, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220936}, {obj_id:8972, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209427}, {obj_id:8973, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220503}, {obj_id:8974, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221262}, {obj_id:8975, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189045}, {obj_id:8976, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194445}, {obj_id:8977, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189493}, {obj_id:8978, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196989}, {obj_id:8979, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190072}, {obj_id:8980, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202973}, {obj_id:8981, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181806}, {obj_id:8982, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180351}, {obj_id:8983, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177512}, {obj_id:8984, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188650}, {obj_id:8985, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179512}]) [2024-08-16 10:20:01.739491] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=224] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:01.744647] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:01.744743] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=53, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:01.744780] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:01.744793] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:01.744802] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:01.744810] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:01.746488] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:01.752198] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.752219] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:01.752230] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.752241] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.752254] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.766575] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=16] ====== tenant freeze timer task ====== [2024-08-16 10:20:01.766605] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=19][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:01.779906] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0C7-0-0] [lt=30] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:01.779953] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0C7-0-0] [lt=24] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:01.779964] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0C7-0-0] [lt=11] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=111470, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:01.779972] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0C7-0-0] [lt=6][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509868, log_task={header:{begin_lsn:{lsn:18544223332}, end_lsn:{lsn:18544223735}, log_id:53509868, min_scn:{val:1723774801578423000, v:0}, max_scn:{val:1723774801578423000, v:0}, data_len:347, proposal_id:7, prev_lsn:{lsn:18544223210}, prev_proposal_id:7, committed_end_lsn:{lsn:18544223332}, data_checksum:1364097841, accum_checksum:1710394436, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774801676684, freeze_ts:1723774801676684, submit_ts:1723774801676694, flushed_ts:1723774801677481, gen_to_freeze cost time:0, gen_to_submit cost time:10, submit_to_flush cost time:787}, fs_cb_begin_ts=1723774801779943, log_life_time=103259) [2024-08-16 10:20:01.780495] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=29] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1379) [2024-08-16 10:20:01.801047] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.801121] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=71] handle task succ(id={id:1260}) [2024-08-16 10:20:01.801139] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.801154] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.801175] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.816170] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=814235648, global_cache_size=97796096, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:01.816296] INFO [COMMON] wash (ob_kvcache_store.cpp:335) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=36] Wash memory, (tenant_id=1002, cache_size=54099968, lower_mem_limit=6442450944, upper_mem_limit=6442450944, min_wash_size=0, max_wash_size=0, mem_usage=2463698944, reserve_mem=355517662, wash_size=450428230) [2024-08-16 10:20:01.816491] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Wash time detail, (compute_wash_size_time=146, refresh_score_time=66, wash_time=46) [2024-08-16 10:20:01.820311] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=38] Cache wash clean map node details(ret=0, clean_node_count=71, clean_time=0, clean_start_pos=2076162, clean_num=62914) [2024-08-16 10:20:01.833939] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=25] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=6) [2024-08-16 10:20:01.844717] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:01.844852] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=32] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:01.844870] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:01.844877] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:01.845465] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1670) [2024-08-16 10:20:01.846573] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=27][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:01.852273] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.852293] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:01.852304] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.852315] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.852329] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.859953] INFO [SERVER] try_reload_schema (ob_server_schema_updater.cpp:440) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=15] schedule fetch new schema task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:01.859978] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:255) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E7E-0-0] [lt=29] [REFRESH_SCHEMA] start to process schema refresh task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:01.859984] INFO [SERVER] do_heartbeat_event (ob_heartbeat.cpp:268) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=29] try reload schema success(schema_version=1722933080671200, refresh_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_ret=0) [2024-08-16 10:20:01.859998] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:280) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E7E-0-0] [lt=17] [REFRESH_SCHEMA] local schema info is newer, no need to refresh schema(ret=0, ret="OB_SUCCESS", local_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:01.861896] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [2421][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=46] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,1,False,{False,},False,False,True}, {1024,1,False,{False,},False,False,True}, {1026,1,False,{False,},False,False,True}, {1028,0,False,{False,},False,False,True}, {1079,1,False,{False,},False,False,True}, {1260,0,False,{False,},False,False,True}, {1261,1,False,{False,},False,False,True}, {1262,1,False,{False,},False,False,True}]) [2024-08-16 10:20:01.880262] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0CD-0-0] [lt=44][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509869, log_task={header:{begin_lsn:{lsn:18544223735}, end_lsn:{lsn:18544223857}, log_id:53509869, min_scn:{val:1723774801578423001, v:0}, max_scn:{val:1723774801578423001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544223332}, prev_proposal_id:7, committed_end_lsn:{lsn:18544223735}, data_checksum:1913811101, accum_checksum:3569938479, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774801779901, freeze_ts:1723774801779901, submit_ts:1723774801779932, flushed_ts:1723774801780538, gen_to_freeze cost time:0, gen_to_submit cost time:31, submit_to_flush cost time:606}, fs_cb_begin_ts=1723774801880247, log_life_time=100346) [2024-08-16 10:20:01.880912] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=597, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:01.885265] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2520][T1002_TenantMet][T1002][Y0-0000000000000000-0-0] [lt=49] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:17374, free_obj_hold_cnt:593, allocator used:71436792, allocator total:73453184}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:1994, free_obj_hold_cnt:6, allocator used:131182000, allocator total:131182000}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():63488, full allocator used:1600, full allocator total:129024}, tablets_mem=204764208, tablets_mem_limit=1288490180, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:7, allocator used:17024, allocator total:65408}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:106, free_obj_hold_cnt:5057, allocator used:10243392, allocator total:10596096}, pending_cnt=0, wait_gc_count=0, tablet count=19368) [2024-08-16 10:20:01.901127] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.901150] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:01.901162] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.901174] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.901189] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.923968] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=8] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:01.944804] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:01.944919] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:01.944939] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:01.944950] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:01.946651] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:01.952101] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=34] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:01.952132] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=23] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:01.952297] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:01.952326] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=20][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:01.952345] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=15][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:01.952346] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:01.952354] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=8][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:01.952360] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] handle task succ(id={id:1260}) [2024-08-16 10:20:01.952368] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:01.952386] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:01.952406] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] try consume task status succ(id={id:1260}) [2024-08-16 10:20:01.955172] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA743C9-0-0] [lt=41] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=12, avg_log_submit_to_flush_time=676, avg_log_submit_to_slide_time=1131) [2024-08-16 10:20:01.961711] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=15] ====== check clog disk timer task ====== [2024-08-16 10:20:01.961734] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=18] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:01.961756] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=8] cannot_recycle_log_size statistics(cannot_recycle_log_size=391646177, threshold=6088116142, need_update_checkpoint_scn=false) [2024-08-16 10:20:01.971587] INFO [CLOG] is_in_sync (ob_log_handler.cpp:546) [31817][T1_L5_G0][T1002][YB42C0A8007C-000616D3CE4B970A-0-0] [lt=24] is_in_sync(ret=0, id=1079, is_log_sync=true, is_need_rebuild=false) [2024-08-16 10:20:01.971720] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007C-000616D3CE4B970A-0-0] [lt=40][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:01.978708] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE60B-0-0] [lt=44] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=13, avg_log_submit_to_flush_time=679, avg_log_submit_to_slide_time=1148) [2024-08-16 10:20:01.978750] INFO [PALF] read (log_cache.cpp:100) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:02.001181] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.001213] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=29] handle task succ(id={id:1260}) [2024-08-16 10:20:02.001252] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=35] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.001266] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.001286] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.012365] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=25] gc stale ls task succ [2024-08-16 10:20:02.016252] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=5] gc stale ls task succ [2024-08-16 10:20:02.016300] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=32] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:02.016323] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=15] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:02.016330] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=6] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:02.016523] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=28] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:02.016569] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:02.021139] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=45] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=812138496, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:02.021592] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=110] Wash time detail, (compute_wash_size_time=411, refresh_score_time=117, wash_time=26) [2024-08-16 10:20:02.026391] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Cache wash clean map node details(ret=0, clean_node_count=61, clean_time=0, clean_start_pos=2139076, clean_num=62914) [2024-08-16 10:20:02.026705] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C16-0-0] [lt=38] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774802026646) [2024-08-16 10:20:02.034067] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=23] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=5) [2024-08-16 10:20:02.044881] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:02.045003] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=30] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1670, avg_group_log_size=167) [2024-08-16 10:20:02.045034] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1670) [2024-08-16 10:20:02.045061] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:02.045106] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=42] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:02.045114] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:02.045860] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31842][BlackListServic][T1][YB42C0A80080-00061CCCC6B436C1-0-0] [lt=42] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12239, add_timestamp:1723774802045841}) [2024-08-16 10:20:02.045915] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676700-0-0] [lt=71] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12239, locations=[{table_id:12239, partition_id:12239, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802045889, sql_renew_time:1723774802045889, is_mark_fail:false}]) [2024-08-16 10:20:02.045944] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31842][BlackListServic][T1][YB42C0A80080-00061CCCC6B436C1-0-0] [lt=27] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12287, add_timestamp:1723774802045941}) [2024-08-16 10:20:02.045954] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676700-0-0] [lt=36] success to process renew task(task={tenant_id:1, table_id:12239, add_timestamp:1723774802045841}, locations=[{table_id:12239, partition_id:12239, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802045889, sql_renew_time:1723774802045889, is_mark_fail:false}]) [2024-08-16 10:20:02.046016] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676701-0-0] [lt=50] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=12287, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:02.046597] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676701-0-0] [lt=17] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12287, locations=[{table_id:12287, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:02.046657] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676701-0-0] [lt=58] success to process renew task(task={tenant_id:1, table_id:12287, add_timestamp:1723774802045941}, locations=[{table_id:12287, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774802046545, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:02.047059] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:02.052447] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.052482] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=33] handle task succ(id={id:1260}) [2024-08-16 10:20:02.052493] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.052503] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.052516] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.053789] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=52] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:02.056595] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80080-00061CCCC6B436C1-0-0] [lt=17][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:02.058212] INFO [STORAGE.TRANS] print_stat_ (ob_black_list.cpp:398) [31842][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=15] start to print blacklist info [2024-08-16 10:20:02.058399] INFO [STORAGE.TRANS] run1 (ob_black_list.cpp:194) [31842][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=22] ls blacklist refresh finish(cost_time=13298) [2024-08-16 10:20:02.065063] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=89][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:02.065107] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=43][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:02.072133] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=31] PNIO [ratelimit] time: 1723774802072131, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007594, add_bytes: 0 [2024-08-16 10:20:02.077577] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=29] PNIO [ratelimit] time: 1723774802077574, bytes: 1496436812442, bw: 0.192822 MB/s, add_ts: 1000778, add_bytes: 202346 [2024-08-16 10:20:02.077616] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD860E-0-0] [lt=52] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:02.078899] INFO [LIB] stat (utility.h:1197) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=34] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=29, stat_interval=1000000, avg (count/cost)=39, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:02.078923] INFO [LIB] stat (utility.h:1197) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=23] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=29, stat_interval=1000000, avg (count/cost)=7, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:02.092361] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0D1-0-0] [lt=23][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509870, log_task={header:{begin_lsn:{lsn:18544223857}, end_lsn:{lsn:18544223979}, log_id:53509870, min_scn:{val:1723774801843482000, v:0}, max_scn:{val:1723774801843482000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544223735}, prev_proposal_id:7, committed_end_lsn:{lsn:18544223857}, data_checksum:1913811101, accum_checksum:1451649999, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774801880226, freeze_ts:1723774801880226, submit_ts:1723774801880231, flushed_ts:1723774801880972, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:741}, fs_cb_begin_ts=1723774802092347, log_life_time=212121) [2024-08-16 10:20:02.093091] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=36, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:20:02.101267] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.101305] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=35] handle task succ(id={id:1260}) [2024-08-16 10:20:02.101323] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.101340] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=15] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.101381] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.110261] INFO [SERVER] fill_tenant_ids_ (ob_all_virtual_tx_stat.cpp:139) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E11-0-0] [lt=15] succeed to get tenant ids(all_tenants_=[1, 1001, 1002]) [2024-08-16 10:20:02.132803] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DC-0-0] [lt=76][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:02.133586] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DC-0-0] [lt=28] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:02.144960] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:02.145093] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:02.145109] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:02.145121] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:02.145785] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1670) [2024-08-16 10:20:02.145729] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=16] get_min_unreplayed_log_info(lsn={lsn:18544223979}, scn={val:1723774801843482001, v:0}, this={ls_id_:{id:1079}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774802092350, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:11904, ret_code_:0}}, next_to_submit_lsn_:{lsn:18544223979}, next_to_submit_scn_:{val:1723774801843482001, v:0}, base_lsn_:{lsn:13018324992}, base_scn_:{val:1720509584063969004, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd005000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:18544223857}, end_lsn:{lsn:18544223979}, read_buf:{buf_len_:2154496, buf_:0x7fa6cd005000}, block_size:67104768, log_storage_:0x7fa6fc6ef470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774801843482000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774801843482000, v:0}, data_checksum:959419206, flag:0}}, init_mode_version:0, accumulate_checksum:1451649999, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774801843482000, v:0}}}}}) [2024-08-16 10:20:02.147116] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=36][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:02.152524] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.152544] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:02.152554] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.152564] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.152577] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.154277] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AD9-0-0] [lt=76] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:02.154302] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AD9-0-0] [lt=23] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:02.154969] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:02.155326] INFO [PALF] read (log_cache.cpp:100) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=35] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:02.161089] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=35] ====== tenant freeze timer task ====== [2024-08-16 10:20:02.179331] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AAF-0-0] [lt=22] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:02.179389] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AAF-0-0] [lt=35] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774801758016003, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774801758016003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:02.183630] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=49] gc stale ls task succ [2024-08-16 10:20:02.185117] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E12-0-0] [lt=36][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=52, len=285, pos=256, bitmap=NULL) [2024-08-16 10:20:02.201358] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.201412] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=51] handle task succ(id={id:1260}) [2024-08-16 10:20:02.201426] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.201438] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.201466] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.226950] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=31] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=814235648, global_cache_size=97796096, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:02.227217] INFO [COMMON] wash (ob_kvcache_store.cpp:335) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=100] Wash memory, (tenant_id=1002, cache_size=54099968, lower_mem_limit=6442450944, upper_mem_limit=6442450944, min_wash_size=0, max_wash_size=0, mem_usage=2463698944, reserve_mem=355517662, wash_size=450428230) [2024-08-16 10:20:02.227418] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash time detail, (compute_wash_size_time=226, refresh_score_time=129, wash_time=67) [2024-08-16 10:20:02.232060] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=24] Cache wash clean map node details(ret=0, clean_node_count=61, clean_time=0, clean_start_pos=2201990, clean_num=62914) [2024-08-16 10:20:02.234195] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=44] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=4) [2024-08-16 10:20:02.235494] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=17] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:02.236169] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=21] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:02.237124] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=29] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:02.237136] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:02.237161] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:02.237188] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=17] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:02.237429] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=23] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:02.245049] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:02.245240] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=56] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:02.245261] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:02.245272] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:02.247115] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=739, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:02.247176] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=29, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:02.247210] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=15, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:02.247211] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:02.247958] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE612-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:02.247976] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE612-0-0] [lt=17] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1394, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:02.252628] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.252657] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:20:02.252673] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.252686] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.252706] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.261180] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=18] re_register_parent reason(palf_id=1262, self="192.168.0.128:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1723774798258127, last_first_register_time_us=1720517744275280, register_time_us=1723774798258127, curr_time_us=1723774802261171) [2024-08-16 10:20:02.261268] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=44] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1262, self="192.168.0.128:2882", reason=1, leader="192.168.0.133:2882", register_time_us=1723774802261226, last_submit_register_req_time_us=1723774802261226, register_parent_reason=1) [2024-08-16 10:20:02.261284] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=15] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1262, self="192.168.0.128:2882") [2024-08-16 10:20:02.261303] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=6] re_register_parent reason(palf_id=1261, self="192.168.0.128:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1723774798258230, last_first_register_time_us=1720517744277428, register_time_us=1723774798258230, curr_time_us=1723774802261303) [2024-08-16 10:20:02.261323] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=8] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1261, self="192.168.0.128:2882", reason=1, leader="192.168.0.125:2882", register_time_us=1723774802261311, last_submit_register_req_time_us=1723774802261311, register_parent_reason=1) [2024-08-16 10:20:02.261331] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=8] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1261, self="192.168.0.128:2882") [2024-08-16 10:20:02.270965] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=30] current monitor number(seq_=-1) [2024-08-16 10:20:02.277772] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8612-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:02.278354] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=566, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:02.278404] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=47] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:02.278439] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=37, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:02.278509] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=17] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=39, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:02.278935] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA743D2-0-0] [lt=21] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=16, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:02.278953] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA743D2-0-0] [lt=17] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1102, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:02.292659] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0DB-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=9, avg_log_batch_cnt=0, total_group_log_size=1379, avg_group_log_size=153) [2024-08-16 10:20:02.292710] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0DB-0-0] [lt=27][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509872, log_task={header:{begin_lsn:{lsn:18544224101}, end_lsn:{lsn:18544224223}, log_id:53509872, min_scn:{val:1723774802154929000, v:0}, max_scn:{val:1723774802154929000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544223979}, prev_proposal_id:7, committed_end_lsn:{lsn:18544224101}, data_checksum:1913811101, accum_checksum:2157951018, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774802192235, freeze_ts:1723774802192235, submit_ts:1723774802192241, flushed_ts:1723774802192988, gen_to_freeze cost time:0, gen_to_submit cost time:6, submit_to_flush cost time:747}, fs_cb_begin_ts=1723774802292693, log_life_time=100458) [2024-08-16 10:20:02.295785] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE81-0-0] [lt=33] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:02.295836] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE81-0-0] [lt=30] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774801863905004, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774801863905004, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:02.299980] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=13] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:20:02.300016] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=32] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=5121, mem_used=39515136) [2024-08-16 10:20:02.301423] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.301445] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:02.301475] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=27] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.301488] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.301509] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.306100] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=16] ====== tenant freeze timer task ====== [2024-08-16 10:20:02.306141] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=25][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:02.324988] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=26] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574632, offset=42145353}, end_cursor:ObLogCursor{file_id=65, log_id=18574633, offset=42145594}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75169c050, len:2487, log_data_len:241, seq:18574632, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:02.325802] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=52] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574633, offset=42145594}, end_cursor:ObLogCursor{file_id=65, log_id=18574634, offset=42145843}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516c8050, len:2246, log_data_len:249, seq:18574633, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:02.327459] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA4-0-0] [lt=20] push gts task success(*task={:""}) [2024-08-16 10:20:02.335034] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA4-0-0] [lt=18] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:02.338032] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=25] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574634, offset=42145843}, end_cursor:ObLogCursor{file_id=65, log_id=18574635, offset=42146092}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516f0050, len:1997, log_data_len:249, seq:18574634, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:02.338807] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=60] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574635, offset=42146092}, end_cursor:ObLogCursor{file_id=65, log_id=18574636, offset=42146341}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516b4050, len:1748, log_data_len:249, seq:18574635, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:02.345145] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:02.345314] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=40] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:02.345355] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=37] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:02.345369] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:02.347294] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=17][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:02.352723] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.352755] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=29] handle task succ(id={id:1260}) [2024-08-16 10:20:02.352773] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.352789] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.352811] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.377888] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8614-0-0] [lt=47] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:02.384228] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=24] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:02.384864] INFO [CLOG] run1 (ob_garbage_collector.cpp:1338) [2765][T1001_GCCollect][T1001][Y0-0000000000000000-0-0] [lt=14] Garbage Collector is running(seq_=325821, gc_interval=10000000) [2024-08-16 10:20:02.384900] INFO [CLOG] gc_check_member_list_ (ob_garbage_collector.cpp:1431) [2765][T1001_GCCollect][T1001][Y0-0000000000000000-0-0] [lt=16] gc_check_member_list_ cost time(ret=0, time_us=19) [2024-08-16 10:20:02.384939] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1703) [2765][T1001_GCCollect][T1001][Y0-0000000000000000-0-0] [lt=14] execute_gc cost time(ret=0, time_us=1) [2024-08-16 10:20:02.384954] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1703) [2765][T1001_GCCollect][T1001][Y0-0000000000000000-0-0] [lt=11] execute_gc cost time(ret=0, time_us=0) [2024-08-16 10:20:02.384961] INFO [SERVER] handle (ob_safe_destroy_handler.cpp:240) [2765][T1001_GCCollect][T1001][Y0-0000000000000000-0-0] [lt=5] ObSafeDestroyHandler start process [2024-08-16 10:20:02.392599] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0E1-0-0] [lt=31] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1379) [2024-08-16 10:20:02.392646] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0E1-0-0] [lt=25] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=9, avg_log_submit_to_flush_time=672, avg_log_submit_to_slide_time=112648) [2024-08-16 10:20:02.392672] INFO [PALF] read (log_cache.cpp:100) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=17] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1379, hit_cnt=9, read_cnt=9, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:02.401504] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.401570] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=63] handle task succ(id={id:1260}) [2024-08-16 10:20:02.401589] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.401605] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.401627] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.405859] INFO pn_ratelimit (group.c:551) [31557][IngressService][T0][Y0-0000000000000000-0-0] [lt=18] PNIO set ratelimit as 9223372036854775807 bytes/s, grp_id=2 [2024-08-16 10:20:02.412955] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=36] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.412996] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=36] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.415162] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=12] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.415215] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=25] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.415248] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=24] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.415260] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.415281] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=10] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.415303] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=16] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.415363] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:02.415379] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.415405] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=12] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.415421] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E14-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.425167] INFO [STORAGE.TRANS] alloc (ob_ts_response_handler.cpp:108) [31439][pnio1][T0][YB42C0A80080-00061CCCCD6206B3-0-0] [lt=26] ts response task statistics(alloc_count=170000486, free_count=85000242) [2024-08-16 10:20:02.425417] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=18] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:02.425555] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=23] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:02.432552] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=53] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=814235648, global_cache_size=97796096, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:02.432684] INFO [COMMON] wash (ob_kvcache_store.cpp:335) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=32] Wash memory, (tenant_id=1002, cache_size=54099968, lower_mem_limit=6442450944, upper_mem_limit=6442450944, min_wash_size=0, max_wash_size=0, mem_usage=2463698944, reserve_mem=355517662, wash_size=450428230) [2024-08-16 10:20:02.432893] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash time detail, (compute_wash_size_time=154, refresh_score_time=67, wash_time=62) [2024-08-16 10:20:02.434307] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=23] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=3) [2024-08-16 10:20:02.436648] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=78, clean_time=0, clean_start_pos=2264904, clean_num=62914) [2024-08-16 10:20:02.445207] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:02.445386] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=36] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:02.445409] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:02.445417] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:02.447372] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=26][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:02.452797] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.452828] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:20:02.452843] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.452867] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=22] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.452895] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.478020] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [2517][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=29] run rewrite rule refresh task(rule_mgr_->tenant_id_=1002) [2024-08-16 10:20:02.479123] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=48] new mysql sessid created(conn.sessid_=3225590104, support_ssl=false) [2024-08-16 10:20:02.479180] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=23] sm conn init succ(conn.sessid_=3225590104, sess.client_addr_="127.0.0.1:51048") [2024-08-16 10:20:02.479202] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=15] accept one succ(*s={this:0x7fa733c7e030, session_id:3225590104, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:119}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:02.479322] INFO [SERVER] extract_user_tenant (obmp_connect.cpp:79) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=18] username and tenantname(user_name=ocp_monitor, tenant_name=) [2024-08-16 10:20:02.479375] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31556][sql_nio1][T1][Y0-0000000000000000-0-0] [lt=27] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:02.479499] INFO load_privilege_info (obmp_connect.cpp:532) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D2F-0-0] [lt=26] no tenant name set, use default tenant name(tenant_name=sys) [2024-08-16 10:20:02.479927] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D2F-0-0] [lt=23][errcode=0] uref leak check(node->uref_=1074538683, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:02.480013] INFO [SERVER] process (obmp_connect.cpp:475) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D2F-0-0] [lt=19] MySQL LOGIN(direct_client_ip="127.0.0.1", client_ip=127.0.0.1, tenant_name=sys, tenant_id=1, user_name=ocp_monitor, host_name=%, sessid=3225590104, proxy_sessid=0, sess_create_time=0, from_proxy=false, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=762509, proxy_capability=0, use_ssl=false, c/s protocol="OB_MYSQL_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:02.481157] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=11] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.481188] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=29] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.481231] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.481252] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=20] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.482152] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=38] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.482178] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=24] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.482692] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007D-00061CCCA5024D1D-0-0] [lt=19][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:02.483052] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=18] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.483114] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=39] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.483139] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=18] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.483164] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=19] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.483179] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=8] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.483187] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.483226] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.483290] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=22] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.483317] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:02.483320] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=22] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.483329] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=11] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.483346] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.483340] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.483358] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E15-0-0] [lt=11] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.483367] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=16] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.483393] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=19] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.483463] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=16] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:02.483480] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=16] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.483498] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.483505] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56165-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.483957] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=12] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.483997] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.484006] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.484010] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.484017] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.484026] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.484067] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=13] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:02.484104] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=36] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.484122] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=8] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.484129] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C79-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.484468] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=14][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa733c7e030, session_id:3225590104, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:3, sql_initiative_shutdown:false, reader:{fd:119}, err:5, last_decode_time:1723774802483668, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:168, pending_flag:1, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:02.484519] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=41] kill and revert session(conn.sessid_=3225590104, proxy_sessid=0, server_id=16, ret=0) [2024-08-16 10:20:02.484529] WDIAG [SERVER] disconnect (obmp_packet_sender.cpp:830) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E16-0-0] [lt=8][errcode=0] server close connection(sessid=3225590104, proxy_sessid=0, stack="0x12a0688c 0xa219970 0xa1e504b 0xa527c01 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:02.484555] INFO [SERVER] process (obmp_quit.h:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E16-0-0] [lt=23] quit [2024-08-16 10:20:02.484562] WDIAG [RPC.OBMYSQL] async_write_data (ob_sql_sock_session.cpp:163) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E16-0-0] [lt=6][errcode=-4009] sock has error(ret=-4009) [2024-08-16 10:20:02.484571] WDIAG [SERVER] flush_buffer (obmp_packet_sender.cpp:1032) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E16-0-0] [lt=8][errcode=-4009] write response fail(ret=-4009) [2024-08-16 10:20:02.484588] WDIAG [SERVER] response (obmp_base.cpp:87) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E16-0-0] [lt=15][errcode=-4009] failed to flush_buffer(ret=-4009) [2024-08-16 10:20:02.484598] WDIAG [RPC.FRAME] run (ob_sql_processor.cpp:48) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E16-0-0] [lt=9][errcode=-4009] response rpc result fail(ret=-4009) [2024-08-16 10:20:02.501599] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.501637] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=35] handle task succ(id={id:1260}) [2024-08-16 10:20:02.501655] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.501669] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.501691] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.505657] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80083-000616E44D0F990D-0-0] [lt=25][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:02.525942] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=51] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=6946, mem_used=12478464) [2024-08-16 10:20:02.533359] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=27] start do ls ha handler(ls_id_array_=[{id:1262}, {id:1260}, {id:1261}, {id:1079}]) [2024-08-16 10:20:02.534008] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31502][AutoLSLoc][T0][YB42C0A80080-00061CCC8EEBBF6C-0-0] [lt=88] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:02.545067] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [31779][T1_ArcSender][T1][YB42C0A80080-00061CCC8D41D0C5-0-0] [lt=25][errcode=-4018] try free send task failed(ret=-4018) [2024-08-16 10:20:02.545292] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:02.545419] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:02.545436] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:02.545447] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:02.547223] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=15] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.547255] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=31] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.547450] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:02.549415] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=14] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.549470] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=27] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.549481] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.549486] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=5] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.549502] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.549527] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=18] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.549590] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=12] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:02.549612] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.549639] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=12] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.549655] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E17-0-0] [lt=15] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.552876] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.552907] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:20:02.552920] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.552932] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.552958] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.554635] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AE1-0-0] [lt=21] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:02.560505] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [2790][T1001_ArcSender][T1001][YB42C0A80080-00061CCC9351D0C5-0-0] [lt=18][errcode=-4018] try free send task failed(ret=-4018) [2024-08-16 10:20:02.562546] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 376.00) [2024-08-16 10:20:02.562580] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=28] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:02.562647] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=32] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.98, rt: 614.00) [2024-08-16 10:20:02.562667] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SHARED_BLOCK_ReaderWriter_IO, mode: read, size: 4191.18, iops: 2442.59, rt: 229.63) [2024-08-16 10:20:02.562676] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: ROOT_BLOCK_IO, mode: read, size: 6144.00, iops: 1.99, rt: 229.00) [2024-08-16 10:20:02.562696] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=3, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:02.562723] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=21] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562732] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562737] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562741] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562745] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562755] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562760] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562764] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562768] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562772] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=10, req_count=1, reservation_ts=1723774802562733, group_limitation_ts=1723774802562733, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562776] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562780] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562787] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562795] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562830] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=33] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.562841] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:02.567392] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [31738][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=86] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:940, allocator used:3737440, allocator total:3859072}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:140, allocator used:9182740, allocator total:9182740}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=13041812, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:63, allocator used:124992, allocator total:130816}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:02.578043] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8618-0-0] [lt=25][errcode=-4389] log_task life cost too much time(palf_id=1261, self="192.168.0.128:2882", log_id=33675590, log_task={header:{begin_lsn:{lsn:32601565398}, end_lsn:{lsn:32601565520}, log_id:33675590, min_scn:{val:1723774802364129001, v:0}, max_scn:{val:1723774802364129001, v:0}, data_len:66, proposal_id:5, prev_lsn:{lsn:32601565276}, prev_proposal_id:5, committed_end_lsn:{lsn:32601565398}, data_checksum:2570081458, accum_checksum:3645154360, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774802477807, freeze_ts:1723774802477807, submit_ts:1723774802477817, flushed_ts:1723774802479151, gen_to_freeze cost time:0, gen_to_submit cost time:10, submit_to_flush cost time:1334}, fs_cb_begin_ts=1723774802578014, log_life_time=100207) [2024-08-16 10:20:02.581140] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=15] can close safely, do destroy(*s={this:0x7fa733c7e030, session_id:3225590104, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:true, reader:{fd:119}, err:5, last_decode_time:1723774802483668, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:173, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-08-16 10:20:02.581204] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=41] unlock session of tenant(conn.sessid_=3225590104, proxy_sessid=0, conn.tenant_id_=1) [2024-08-16 10:20:02.581255] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21114][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=6][errcode=0] uref leak check(node->uref_=1074538683, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:02.581228] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=21] connection close(sessid=3225590104, proxy_sessid=0, tenant_id=1, server_id=16, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCBC524C79-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:8, pkt_rec[0]:{send:obp_mysql_header_{len_:11, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{receive:obp_mysql_header_{len_:63, rec_:63, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[2]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{send:obp_mysql_header_{pkt_num_:2, seq_:4}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[4]:{send:obp_mysql_header_{len_:9, seq_:5}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_mysql_header_{pkt_num_:3, seq_:8}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[6]:{send:obp_mysql_header_{len_:9, seq_:9}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[7]:{receive:obp_mysql_header_{len_:1, rec_:1, seq_:0}, pkt_name:"COM_QUIT", obp_mysql_header_.type_:1, obp_mysql_header_.is_send_:0}], disconnect_state=3) [2024-08-16 10:20:02.581331] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=67] new mysql sessid created(conn.sessid_=3225521020, support_ssl=false) [2024-08-16 10:20:02.581347] INFO [SERVER] run (obmp_disconnect.cpp:82) [21114][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=34] free session successfully(sessid=3225590104) [2024-08-16 10:20:02.581364] INFO [SERVER] run (obmp_disconnect.cpp:89) [21114][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=15] mark session id unused(sessid=3225590104) [2024-08-16 10:20:02.581385] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=18] sm conn init succ(conn.sessid_=3225521020, sess.client_addr_="127.0.0.1:51050") [2024-08-16 10:20:02.581407] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=15] accept one succ(*s={this:0x7fa70d9fed70, session_id:3225521020, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:155}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:02.581749] INFO [SERVER] extract_user_tenant (obmp_connect.cpp:79) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=25] username and tenantname(user_name=ocp_monitor, tenant_name=) [2024-08-16 10:20:02.581788] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31556][sql_nio1][T1][Y0-0000000000000000-0-0] [lt=20] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:02.581889] INFO load_privilege_info (obmp_connect.cpp:532) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24C-0-0] [lt=31] no tenant name set, use default tenant name(tenant_name=sys) [2024-08-16 10:20:02.582315] WDIAG [COMMON] get (ob_link_hashmap.h:313) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24C-0-0] [lt=20][errcode=0] uref leak check(node->uref_=1074538684, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:02.582400] INFO [SERVER] process (obmp_connect.cpp:475) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24C-0-0] [lt=38] MySQL LOGIN(direct_client_ip="127.0.0.1", client_ip=127.0.0.1, tenant_name=sys, tenant_id=1, user_name=ocp_monitor, host_name=%, sessid=3225521020, proxy_sessid=0, sess_create_time=0, from_proxy=false, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=762509, proxy_capability=0, use_ssl=false, c/s protocol="OB_MYSQL_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:02.583311] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=26] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.583312] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=8] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.583329] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=17] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.583334] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.584230] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=19] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.584254] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=23] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.585098] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=15] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.585143] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.585152] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.585157] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.585165] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.585169] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=3] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.585225] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=4] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:02.585231] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.585245] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.585248] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E18-0-0] [lt=3] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.585305] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=11] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.585352] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=27] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.585361] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=8] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.585371] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.585390] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=13] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.585397] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.585491] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=3] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:02.585507] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=15] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.585548] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=32] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.585555] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21122][T1_L0_G0][T1][YB42C0A80080-00061CCCC2A1F607-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.586149] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=10] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.586212] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=25] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.586222] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:02.586233] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.586250] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=11] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.586260] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=8] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.586321] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=19] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:02.586357] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=34] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.586386] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=16] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:02.586404] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56167-0-0] [lt=17] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:02.586800] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa70d9fed70, session_id:3225521020, trace_id:YB42C0A80080-00061CCCC65F2E19-0-0, sql_handling_stage:10, sql_initiative_shutdown:false, reader:{fd:155}, err:5, last_decode_time:1723774802585885, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:168, pending_flag:1, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:02.586815] WDIAG [SERVER] disconnect (obmp_packet_sender.cpp:830) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E19-0-0] [lt=4][errcode=0] server close connection(sessid=3225521020, proxy_sessid=0, stack="0x12a0688c 0xa219970 0xa1e504b 0xa527c01 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:02.586835] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=28] kill and revert session(conn.sessid_=3225521020, proxy_sessid=0, server_id=16, ret=0) [2024-08-16 10:20:02.586836] INFO [SERVER] process (obmp_quit.h:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E19-0-0] [lt=17] quit [2024-08-16 10:20:02.586846] WDIAG [RPC.OBMYSQL] async_write_data (ob_sql_sock_session.cpp:163) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E19-0-0] [lt=8][errcode=-4009] sock has error(ret=-4009) [2024-08-16 10:20:02.586854] WDIAG [SERVER] flush_buffer (obmp_packet_sender.cpp:1032) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E19-0-0] [lt=7][errcode=-4009] write response fail(ret=-4009) [2024-08-16 10:20:02.586860] WDIAG [SERVER] response (obmp_base.cpp:87) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E19-0-0] [lt=5][errcode=-4009] failed to flush_buffer(ret=-4009) [2024-08-16 10:20:02.586867] WDIAG [RPC.FRAME] run (ob_sql_processor.cpp:48) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E19-0-0] [lt=6][errcode=-4009] response rpc result fail(ret=-4009) [2024-08-16 10:20:02.591273] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=13][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:02.592113] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] ====== checkpoint timer task ====== [2024-08-16 10:20:02.592808] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0E7-0-0] [lt=82][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509874, log_task={header:{begin_lsn:{lsn:18544224345}, end_lsn:{lsn:18544224467}, log_id:53509874, min_scn:{val:1723774802382403000, v:0}, max_scn:{val:1723774802382403000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544224223}, prev_proposal_id:7, committed_end_lsn:{lsn:18544224345}, data_checksum:1913811101, accum_checksum:406975026, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774802392547, freeze_ts:1723774802392547, submit_ts:1723774802392585, flushed_ts:1723774802393229, gen_to_freeze cost time:0, gen_to_submit cost time:38, submit_to_flush cost time:644}, fs_cb_begin_ts=1723774802592790, log_life_time=200243) [2024-08-16 10:20:02.593168] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=17] succ to get rec scn(*this={this:0x7fa707804030, ls_id:{id:1079}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1722510647991558001, v:0}, last_push_gc_task_ts_:1722511534898926, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:103}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:02.593208] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=38] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596c080, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1722511534901545, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596c080, snapshot_version:{val:1722511534901545, v:0}, ls_id:{id:1079}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:02.593246] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=29] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1722497541268834001, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1722497541268834001, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1079}) [2024-08-16 10:20:02.593324] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:265) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=28] [FREEZE]get rec_scn from MdsTableMgr(ret=0, ret="OB_SUCCESS", min_rec_scn={val:1723723445809886004, v:0}, order_flusher_for_one={high_priority_flusher:size:1, ordered_tablets:{{tablet_id:{id:208204}, rec_scn:{val:1723723445809886004, v:0}}, }, *this={this:0x7fa6fa246510, is_inited:true, freezing_scn:{val:1721997282211836001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774800655290003, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:02.593435] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=105] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="MDS_TABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={this:0x7fa6fa246510, is_inited:true, freezing_scn:{val:1721997282211836001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774800655290003, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}, min_rec_scn={val:1723723445809886004, v:0}, ls_id_={id:1079}) [2024-08-16 10:20:02.593511] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=62] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723723202639465000, v:0}, checkpoint_scn_in_ls_meta={val:1723723202639465000, v:0}, ls_id={id:1079}, service_type="DUP_TABLE") [2024-08-16 10:20:02.593538] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=11][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:18453811200}, new_base_lsn={lsn:18453811200}, lsn={lsn:18453811200}) [2024-08-16 10:20:02.593550] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=11] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:18453811200}, ls->get_ls_id()={id:1079}) [2024-08-16 10:20:02.593578] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=22] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:02.593578] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=18, stat_interval=1000000, avg (count/cost)=31, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:02.593665] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=23][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:02.593692] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=25][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:02.593705] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=11] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:02.594426] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] succ to get rec scn(*this={this:0x7fa7078b6030, ls_id:{id:1260}, tenant_id:1002, state:"L_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1723690101339443000, v:0}, last_push_gc_task_ts_:1723691902664029, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:6460}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:02.594452] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=25] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596eab0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1723691902666264, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596eab0, snapshot_version:{val:1723691902666264, v:0}, ls_id:{id:1260}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:02.594470] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=14] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1723690100831412000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1723690100831412000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1260}) [2024-08-16 10:20:02.596234] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:265) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] [FREEZE]get rec_scn from MdsTableMgr(ret=0, ret="OB_SUCCESS", min_rec_scn={val:1723723302639709002, v:0}, order_flusher_for_one={high_priority_flusher:size:1, ordered_tablets:{{tablet_id:{id:1152921504606868225}, rec_scn:{val:1723723302639709002, v:0}}, }, *this={this:0x7fa6bd53e510, is_inited:true, freezing_scn:{val:1723690100831412000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1260}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202608224000, v:0}, clog_base_lsn:{lsn:8723619840}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774800655290003, v:0}, tablet_change_checkpoint_scn:{val:1723719123293042000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723639313794159001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1260, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:02.596293] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=55] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x7fa6bd53e1b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x7fa6bd543cd0, table_type:1, memtable_head:149, memtable_tail:150, t3m:0x7fa7118b2030, tables:[null, null, null, null, null, 0x7fa70dc06cc0, null, null, null, null, null, null, null, null, null, null]}, is_freezing:false, ls_id:{id:1260}, mini_merge_recycle_commit_versions_ts:1723723202700999, tx_data_table:0x7fa6bd5451d0, ls_tablet_svr:0x7fa6bd53e190, slice_allocator:0x7fa6bd545210}, min_rec_scn={val:1723723202608224000, v:0}, ls_id_={id:1260}) [2024-08-16 10:20:02.601665] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=34] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.601689] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:02.601700] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.601710] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.601724] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.601817] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=34] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723723202608224000, v:0}, checkpoint_scn_in_ls_meta={val:1723723202608224000, v:0}, ls_id={id:1260}, service_type="TRANS_SERVICE") [2024-08-16 10:20:02.601853] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:8723619840}, new_base_lsn={lsn:8723619840}, lsn={lsn:8723619840}) [2024-08-16 10:20:02.601869] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=14] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:8723619840}, ls->get_ls_id()={id:1260}) [2024-08-16 10:20:02.602825] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] succ to get rec scn(*this={this:0x7fa70da04030, ls_id:{id:1261}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1723712452247151000, v:0}, last_push_gc_task_ts_:1723712455858054, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:6477}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:02.602858] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=31] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596ce90, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1723723202513573, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596ce90, snapshot_version:{val:1723723202513573, v:0}, ls_id:{id:1261}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:02.602890] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=26] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1723654533322558000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1723654533322558000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1261}) [2024-08-16 10:20:02.603477] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=21] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x7fa6fe204510, is_inited:true, freezing_scn:{val:1723752041922970000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1261}, ls_create_status:1, clog_checkpoint_scn:{val:1723752188579788000, v:0}, clog_base_lsn:{lsn:32545812480}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774800655290003, v:0}, tablet_change_checkpoint_scn:{val:1723719128263216001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723617733349446001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:4, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1261}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1261, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:02.603563] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=82] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x7fa6fe2041b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x7fa6fe209cd0, table_type:1, memtable_head:173, memtable_tail:174, t3m:0x7fa7118b2030, tables:[null, null, null, null, null, null, null, null, null, null, null, null, null, 0x7fa70dc09900, null, null]}, is_freezing:false, ls_id:{id:1261}, mini_merge_recycle_commit_versions_ts:1723752042418277, tx_data_table:0x7fa6fe20b1d0, ls_tablet_svr:0x7fa6fe204190, slice_allocator:0x7fa6fe20b210}, min_rec_scn={val:1723752188579788000, v:0}, ls_id_={id:1261}) [2024-08-16 10:20:02.607263] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=31] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:02.609359] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=25] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723752188579788000, v:0}, checkpoint_scn_in_ls_meta={val:1723752188579788000, v:0}, ls_id={id:1261}, service_type="TRANS_SERVICE") [2024-08-16 10:20:02.609400] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=28][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:32545812480}, new_base_lsn={lsn:32545812480}, lsn={lsn:32545812480}) [2024-08-16 10:20:02.609414] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=13] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:32545812480}, ls->get_ls_id()={id:1261}) [2024-08-16 10:20:02.610300] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] succ to get rec scn(*this={this:0x7fa70dab6030, ls_id:{id:1262}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1723712432019955000, v:0}, last_push_gc_task_ts_:1723712435838837, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:6328}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:02.610328] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=27] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596dca0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1723712435841010, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596dca0, snapshot_version:{val:1723712435841010, v:0}, ls_id:{id:1262}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:02.610351] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=18] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1723657339856816000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1723657339856816000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1262}) [2024-08-16 10:20:02.611909] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:265) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=18] [FREEZE]get rec_scn from MdsTableMgr(ret=0, ret="OB_SUCCESS", min_rec_scn={val:1723723207973113002, v:0}, order_flusher_for_one={high_priority_flusher:size:1, ordered_tablets:{{tablet_id:{id:209134}, rec_scn:{val:1723723207973113002, v:0}}, }, *this={this:0x7fa6be4fc510, is_inited:true, freezing_scn:{val:1723656976361279000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774800655290003, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:02.611984] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=71] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="MDS_TABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={this:0x7fa6be4fc510, is_inited:true, freezing_scn:{val:1723656976361279000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774800655290003, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}, min_rec_scn={val:1723723207973113002, v:0}, ls_id_={id:1262}) [2024-08-16 10:20:02.617548] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=29] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=1, size_used=60687, mem_used=137263104) [2024-08-16 10:20:02.617739] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=40] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723723207973113002, v:0}, checkpoint_scn_in_ls_meta={val:1723723207973113002, v:0}, ls_id={id:1262}, service_type="TRANS_SERVICE") [2024-08-16 10:20:02.617776] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=24][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:8253886464}, new_base_lsn={lsn:8253886464}, lsn={lsn:8253886464}) [2024-08-16 10:20:02.617796] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=18] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:8253886464}, ls->get_ls_id()={id:1262}) [2024-08-16 10:20:02.617808] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:191) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] succeed to update_clog_checkpoint(ret=0, ls_cnt=4) [2024-08-16 10:20:02.627281] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1044) [2747][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=22] success to update min reserved snapshot(reserved_snapshot=1723772990629965000, duration=1800, snapshot_gc_ts_=1723774790629965000) [2024-08-16 10:20:02.627321] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1071) [2747][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=32] update reserved snapshot finished(cost_ts=45, reserved_snapshot=1723772990629965000) [2024-08-16 10:20:02.634417] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=27] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=2) [2024-08-16 10:20:02.636594] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=44] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:02.637170] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=23] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=812138496, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:02.637462] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash time detail, (compute_wash_size_time=152, refresh_score_time=72, wash_time=25) [2024-08-16 10:20:02.641604] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=89, clean_time=0, clean_start_pos=2327818, clean_num=62914) [2024-08-16 10:20:02.645189] INFO [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:262) [31779][T1_ArcSender][T1][YB42C0A80080-00061CCC8D41D0C5-0-0] [lt=37] ObArchiveSender is running(thread_index=0) [2024-08-16 10:20:02.645378] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:02.645525] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:02.645545] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:02.645552] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:02.646803] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE61D-0-0] [lt=53] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=6, avg_log_gen_to_submit_time=75, avg_log_submit_to_flush_time=848, avg_log_submit_to_slide_time=1327) [2024-08-16 10:20:02.646845] INFO [LIB] stat (utility.h:1197) [19129][T1002_ApplySrv2][T1002][YB42C0A80080-00061CCCC39FE198-0-0] [lt=11] [APPLY STAT CB APPEND COST TIME](cur_stat_count=49, stat_interval=5000000, avg (count/cost)=95, this=0x7fa70bda0280, extra_info="") [2024-08-16 10:20:02.646875] INFO [LIB] stat (utility.h:1197) [19129][T1002_ApplySrv2][T1002][YB42C0A80080-00061CCCC39FE198-0-0] [lt=29] [APPLY STAT CB IN QUEUE TIME](cur_stat_count=49, stat_interval=5000000, avg (count/cost)=1301, this=0x7fa70bda04b0, extra_info="") [2024-08-16 10:20:02.646885] INFO [LIB] stat (utility.h:1197) [19129][T1002_ApplySrv2][T1002][YB42C0A80080-00061CCCC39FE198-0-0] [lt=9] [APPLY STAT CB WAIT COMMIT TIME](cur_stat_count=49, stat_interval=5000000, avg (count/cost)=0, this=0x7fa70bda06e0, extra_info="") [2024-08-16 10:20:02.646892] INFO [LIB] stat (utility.h:1197) [19129][T1002_ApplySrv2][T1002][YB42C0A80080-00061CCCC39FE198-0-0] [lt=7] [APPLY STAT CB EXECUTE TIME](cur_stat_count=49, stat_interval=5000000, avg (count/cost)=-539, this=0x7fa70bda0910, extra_info="") [2024-08-16 10:20:02.646923] INFO [LIB] stat (utility.h:1197) [19129][T1002_ApplySrv2][T1002][YB42C0A80080-00061CCCC39FE198-0-0] [lt=30] [APPLY STAT CB TOTAL TIME](cur_stat_count=49, stat_interval=5000000, avg (count/cost)=857, this=0x7fa70bda0b40, extra_info="") [2024-08-16 10:20:02.647518] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=35][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:02.652945] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.652967] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:02.652978] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.652988] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.653001] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.655391] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=608, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:02.655506] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=40, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:02.655722] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F735EA-0-0] [lt=25] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=14, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:02.655741] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F735EA-0-0] [lt=18] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1075, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:02.677870] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_table_mgr.cpp:80) [32373][T1_PX_G0][T1][YB42C0A80080-00061CCCC85CFAE7-0-0] [lt=10][errcode=-4008] fail to execute(ret=-4008) [2024-08-16 10:20:02.677922] WDIAG [SQL.ENG] inner_get_next_row (ob_px_dist_transmit_op.cpp:73) [32373][T1_PX_G0][T1][YB42C0A80080-00061CCCC85CFAE7-0-0] [lt=28][errcode=-4008] fail to exec ObPxTransmitOp::inner_get_next_row()(ret=-4008) [2024-08-16 10:20:02.680252] WDIAG [SERVER] disconnect (obmp_packet_sender.cpp:830) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE8-0-0] [lt=6][errcode=0] server close connection(sessid=3225645364, proxy_sessid=0, stack="0x12a0688c 0xa219970 0xa1e504b 0xa527c01 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:02.680273] INFO [SERVER] process (obmp_quit.h:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE8-0-0] [lt=18] quit [2024-08-16 10:20:02.680282] WDIAG [RPC.OBMYSQL] async_write_data (ob_sql_sock_session.cpp:163) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE8-0-0] [lt=7][errcode=-4009] sock has error(ret=-4009) [2024-08-16 10:20:02.680288] WDIAG [SERVER] flush_buffer (obmp_packet_sender.cpp:1032) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE8-0-0] [lt=6][errcode=-4009] write response fail(ret=-4009) [2024-08-16 10:20:02.680293] WDIAG [SERVER] response (obmp_base.cpp:87) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE8-0-0] [lt=4][errcode=-4009] failed to flush_buffer(ret=-4009) [2024-08-16 10:20:02.680299] WDIAG [RPC.FRAME] run (ob_sql_processor.cpp:48) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAE8-0-0] [lt=5][errcode=-4009] response rpc result fail(ret=-4009) [2024-08-16 10:20:02.680261] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=100][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa71f01e030, session_id:3225645364, trace_id:YB42C0A80080-00061CCCC85CFAE8-0-0, sql_handling_stage:10, sql_initiative_shutdown:true, reader:{fd:129}, err:5, last_decode_time:1723774802679585, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:7383, pending_flag:1, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:02.680326] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=56] kill and revert session(conn.sessid_=3225645364, proxy_sessid=0, server_id=16, ret=0) [2024-08-16 10:20:02.680344] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=17] can close safely, do destroy(*s={this:0x7fa71f01e030, session_id:3225645364, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:true, reader:{fd:129}, err:5, last_decode_time:1723774802679585, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:7388, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-08-16 10:20:02.680368] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=12] unlock session of tenant(conn.sessid_=3225645364, proxy_sessid=0, conn.tenant_id_=1) [2024-08-16 10:20:02.680416] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=11][errcode=0] uref leak check(node->uref_=1074538684, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:02.680383] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=13] connection close(sessid=3225645364, proxy_sessid=0, tenant_id=1, server_id=16, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCC85CFAE7-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:128, pkt_rec[96]:{send:obp_mysql_header_{len_:9, seq_:11}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[97]:{receive:obp_mysql_header_{len_:59, rec_:469, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[98]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[99]:{send:obp_mysql_header_{pkt_num_:185, seq_:3}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[100]:{send:obp_mysql_header_{len_:9, seq_:4}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[101]:{send:obp_mysql_header_{pkt_num_:62, seq_:5}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[102]:{send:obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[103]:{receive:obp_mysql_header_{len_:149, rec_:333, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[104]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[105]:{send:obp_mysql_header_{pkt_num_:511, seq_:3}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[106]:{send:obp_mysql_header_{len_:9, seq_:4}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[107]:{send:obp_mysql_header_{pkt_num_:157, seq_:5}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[108]:{send:obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[109]:{receive:obp_mysql_header_{len_:178, rec_:688, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[110]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[111]:{send:obp_mysql_header_{pkt_num_:386, seq_:5}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[112]:{send:obp_mysql_header_{len_:9, seq_:7}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[113]:{receive:obp_mysql_header_{len_:556, rec_:739, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[114]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[115]:{send:obp_mysql_header_{pkt_num_:393, seq_:6}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[116]:{send:obp_mysql_header_{len_:9, seq_:8}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[117]:{receive:obp_mysql_header_{len_:183, rec_:624, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[118]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[119]:{send:obp_mysql_header_{pkt_num_:377, seq_:7}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[120]:{send:obp_mysql_header_{len_:9, seq_:8}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[121]:{send:obp_mysql_header_{pkt_num_:452, seq_:11}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[122]:{send:obp_mysql_header_{len_:9, seq_:12}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[123]:{receive:obp_mysql_header_{len_:315, rec_:948, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[124]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[125]:{send:obp_mysql_header_{pkt_num_:294, seq_:4}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[126]:{send:obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[127]:{receive:obp_mysql_header_{len_:1, rec_:634, seq_:0}, pkt_name:"COM_QUIT", obp_mysql_header_.type_:1, obp_mysql_header_.is_send_:0}], disconnect_state=3) [2024-08-16 10:20:02.680476] INFO [SERVER] run (obmp_disconnect.cpp:82) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=11] free session successfully(sessid=3225645364) [2024-08-16 10:20:02.680486] INFO [SERVER] run (obmp_disconnect.cpp:89) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=9] mark session id unused(sessid=3225645364) [2024-08-16 10:20:02.692945] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0ED-0-0] [lt=8][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509875, log_task={header:{begin_lsn:{lsn:18544224467}, end_lsn:{lsn:18544224589}, log_id:53509875, min_scn:{val:1723774802478482000, v:0}, max_scn:{val:1723774802478482000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544224345}, prev_proposal_id:7, committed_end_lsn:{lsn:18544224467}, data_checksum:1913811101, accum_checksum:3338866296, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774802592757, freeze_ts:1723774802592757, submit_ts:1723774802592770, flushed_ts:1723774802593584, gen_to_freeze cost time:0, gen_to_submit cost time:13, submit_to_flush cost time:814}, fs_cb_begin_ts=1723774802692933, log_life_time=100176) [2024-08-16 10:20:02.701781] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.701807] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:20:02.701818] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.701827] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.701839] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.745462] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] try gc retain ctx [2024-08-16 10:20:02.745509] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=30] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:02.745632] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=57, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:02.745672] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:02.745682] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:02.745690] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:02.745698] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:02.747531] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=11][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:02.753023] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.753046] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:02.753059] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.753096] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=35] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.753118] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.792904] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0F3-0-0] [lt=58] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=8, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:02.792983] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0F3-0-0] [lt=49] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=8, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:02.792995] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0F3-0-0] [lt=12] [PALF STAT LOG LIFE TIME](cur_stat_count=8, stat_interval=1000000, avg (count/cost)=126668, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:02.793005] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0F3-0-0] [lt=7][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509876, log_task={header:{begin_lsn:{lsn:18544224589}, end_lsn:{lsn:18544224711}, log_id:53509876, min_scn:{val:1723774802478482001, v:0}, max_scn:{val:1723774802478482001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544224467}, prev_proposal_id:7, committed_end_lsn:{lsn:18544224589}, data_checksum:1913811101, accum_checksum:2786874133, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774802692892, freeze_ts:1723774802692892, submit_ts:1723774802692900, flushed_ts:1723774802693641, gen_to_freeze cost time:0, gen_to_submit cost time:8, submit_to_flush cost time:741}, fs_cb_begin_ts=1723774802792970, log_life_time=100078) [2024-08-16 10:20:02.793652] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=976) [2024-08-16 10:20:02.801846] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.801868] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:02.801884] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.801898] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.801923] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.834527] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=25] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=1) [2024-08-16 10:20:02.842094] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:02.842372] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=36] Wash time detail, (compute_wash_size_time=180, refresh_score_time=70, wash_time=16) [2024-08-16 10:20:02.845551] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:02.845684] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=32] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:02.845705] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:02.845716] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:02.845854] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=111, clean_time=0, clean_start_pos=2390732, clean_num=62914) [2024-08-16 10:20:02.846477] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:02.847602] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=46][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:02.853098] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.853128] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=28] handle task succ(id={id:1260}) [2024-08-16 10:20:02.853138] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.853147] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.853161] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.869860] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [31770][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=43] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1, server_version={version:{val:1723774797924931000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0}, server_version_delta=4944924, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.126:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29977666, succ_cluster_heartbeat_count=29977648, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774342455102000, v:0}, local_cluster_delta=460414753, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:20:02.876544] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [2536][T1002_LockWaitM][T1002][Y0-0000000000000000-0-0] [lt=20] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-08-16 10:20:02.893568] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=40] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=673, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:02.901935] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.901963] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=26] handle task succ(id={id:1260}) [2024-08-16 10:20:02.901976] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.901989] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.902006] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.909442] INFO [SHARE] blacklist_loop_ (ob_server_blacklist.cpp:313) [31526][Blacklist][T0][Y0-0000000000000000-0-0] [lt=20] blacklist_loop exec finished(cost_time=24, is_enabled=true, send_cnt=0) [2024-08-16 10:20:02.925296] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=32] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:02.945628] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:02.945784] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=34] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:02.945802] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:02.945809] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:02.947677] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:02.953191] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:02.953225] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=30] handle task succ(id={id:1260}) [2024-08-16 10:20:02.953240] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:02.953254] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:02.953279] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] try consume task status succ(id={id:1260}) [2024-08-16 10:20:02.956235] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA743E6-0-0] [lt=51] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=11, avg_log_submit_to_flush_time=720, avg_log_submit_to_slide_time=1123) [2024-08-16 10:20:02.969702] INFO [SQL.DTL] runTimerTask (ob_dtl_interm_result_manager.cpp:45) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=5] dump dtl interm result cost(us)(dump_cost=23756, ret=0, interm count=0, dump count=324507) [2024-08-16 10:20:02.979716] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE628-0-0] [lt=30] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=11, avg_log_submit_to_flush_time=741, avg_log_submit_to_slide_time=11031) [2024-08-16 10:20:02.979734] INFO [PALF] read (log_cache.cpp:100) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:02.990003] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=8] can close safely, do destroy(*s={this:0x7fa70d9fed70, session_id:3225521020, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:true, reader:{fd:155}, err:5, last_decode_time:1723774802585885, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:173, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-08-16 10:20:02.990061] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=40] unlock session of tenant(conn.sessid_=3225521020, proxy_sessid=0, conn.tenant_id_=1) [2024-08-16 10:20:02.990111] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21114][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=13][errcode=0] uref leak check(node->uref_=1074538685, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:02.990088] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=25] connection close(sessid=3225521020, proxy_sessid=0, tenant_id=1, server_id=16, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCC1B56167-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:8, pkt_rec[0]:{send:obp_mysql_header_{len_:11, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{receive:obp_mysql_header_{len_:63, rec_:63, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[2]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{send:obp_mysql_header_{pkt_num_:2, seq_:4}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[4]:{send:obp_mysql_header_{len_:9, seq_:5}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_mysql_header_{pkt_num_:3, seq_:8}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[6]:{send:obp_mysql_header_{len_:9, seq_:9}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[7]:{receive:obp_mysql_header_{len_:1, rec_:1, seq_:0}, pkt_name:"COM_QUIT", obp_mysql_header_.type_:1, obp_mysql_header_.is_send_:0}], disconnect_state=3) [2024-08-16 10:20:02.990191] INFO [SERVER] run (obmp_disconnect.cpp:82) [21114][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=19] free session successfully(sessid=3225521020) [2024-08-16 10:20:02.990216] INFO [SERVER] run (obmp_disconnect.cpp:89) [21114][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=24] mark session id unused(sessid=3225521020) [2024-08-16 10:20:02.990353] INFO [SERVER] fill_tenant_ids_ (ob_all_virtual_tx_stat.cpp:139) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E1A-0-0] [lt=4] succeed to get tenant ids(all_tenants_=[1, 1001, 1002]) [2024-08-16 10:20:02.992505] INFO [SQL.DTL] runTimerTask (ob_dtl_interm_result_manager.cpp:60) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=43] clear dtl interm result cost(us)(clear_cost=22755, ret=0, expire_keys_.count()=0, dump count=324507, interm count=0, clean count=0) [2024-08-16 10:20:02.992593] INFO [SERVER] update_tenant_wa_percentage (ob_tenant_duty_task.cpp:243) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=24] set tenant work area memory(tenant_id=1001, wa_pctg=5, limit=157286400, ret=0) [2024-08-16 10:20:02.992671] INFO [SERVER] update_tenant_wa_percentage (ob_tenant_duty_task.cpp:243) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=18] set tenant work area memory(tenant_id=1002, wa_pctg=60, limit=3865470540, ret=0) [2024-08-16 10:20:02.993128] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF0FF-0-0] [lt=44][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509879, log_task={header:{begin_lsn:{lsn:18544225236}, end_lsn:{lsn:18544225358}, log_id:53509879, min_scn:{val:1723774802883195000, v:0}, max_scn:{val:1723774802883195000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544224833}, prev_proposal_id:7, committed_end_lsn:{lsn:18544225236}, data_checksum:1913811101, accum_checksum:3874443627, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774802892855, freeze_ts:1723774802892855, submit_ts:1723774802892862, flushed_ts:1723774802893634, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:772}, fs_cb_begin_ts=1723774802993114, log_life_time=100259) [2024-08-16 10:20:02.995900] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=15] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:02.995928] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=21] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:02.996037] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:02.996057] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=14][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:02.996067] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=7][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:02.996115] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=46][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:03.002027] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.002052] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:20:03.002064] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.002090] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.002107] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.012443] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=29] gc stale ls task succ [2024-08-16 10:20:03.012695] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=8] gc stale ls task succ [2024-08-16 10:20:03.012749] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=26] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:03.012781] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=22] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:03.012788] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=6] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:03.016593] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:03.016595] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=24] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:03.019966] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E1B-0-0] [lt=27] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12206, add_timestamp:1723774803019951}) [2024-08-16 10:20:03.020316] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676702-0-0] [lt=37] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12206, locations=[{table_id:12206, partition_id:12206, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803020298, sql_renew_time:1723774803020298, is_mark_fail:false}]) [2024-08-16 10:20:03.020351] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676702-0-0] [lt=33] success to process renew task(task={tenant_id:1, table_id:12206, add_timestamp:1723774803019951}, locations=[{table_id:12206, partition_id:12206, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803020298, sql_renew_time:1723774803020298, is_mark_fail:false}]) [2024-08-16 10:20:03.024124] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=42] new mysql sessid created(conn.sessid_=3225503935, support_ssl=false) [2024-08-16 10:20:03.024187] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=35] sm conn init succ(conn.sessid_=3225503935, sess.client_addr_="127.0.0.1:51052") [2024-08-16 10:20:03.024212] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=94] new mysql sessid created(conn.sessid_=3225599215, support_ssl=false) [2024-08-16 10:20:03.024210] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=16] accept one succ(*s={this:0x7fa733c7e030, session_id:3225503935, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:119}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:03.024248] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=12] sm conn init succ(conn.sessid_=3225599215, sess.client_addr_="127.0.0.1:51054") [2024-08-16 10:20:03.024270] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=16] accept one succ(*s={this:0x7fa71f1dc030, session_id:3225599215, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:129}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:03.024321] INFO [SERVER] extract_user_tenant (obmp_connect.cpp:79) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=14] username and tenantname(user_name=ocp_monitor, tenant_name=) [2024-08-16 10:20:03.024354] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31555][sql_nio0][T1][Y0-0000000000000000-0-0] [lt=15] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:03.024374] INFO [SERVER] extract_user_tenant (obmp_connect.cpp:79) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=17] username and tenantname(user_name=ocp_monitor, tenant_name=) [2024-08-16 10:20:03.024403] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31556][sql_nio1][T1][Y0-0000000000000000-0-0] [lt=11] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:03.024478] INFO load_privilege_info (obmp_connect.cpp:532) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D30-0-0] [lt=36] no tenant name set, use default tenant name(tenant_name=sys) [2024-08-16 10:20:03.024506] INFO load_privilege_info (obmp_connect.cpp:532) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24D-0-0] [lt=30] no tenant name set, use default tenant name(tenant_name=sys) [2024-08-16 10:20:03.024980] WDIAG [COMMON] get (ob_link_hashmap.h:313) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24D-0-0] [lt=9][errcode=0] uref leak check(node->uref_=1074538687, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:03.024987] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D30-0-0] [lt=24][errcode=0] uref leak check(node->uref_=1074538687, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:03.025064] INFO [SERVER] process (obmp_connect.cpp:475) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D30-0-0] [lt=15] MySQL LOGIN(direct_client_ip="127.0.0.1", client_ip=127.0.0.1, tenant_name=sys, tenant_id=1, user_name=ocp_monitor, host_name=%, sessid=3225599215, proxy_sessid=0, sess_create_time=0, from_proxy=false, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=762509, proxy_capability=0, use_ssl=false, c/s protocol="OB_MYSQL_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:03.025064] INFO [SERVER] process (obmp_connect.cpp:475) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24D-0-0] [lt=27] MySQL LOGIN(direct_client_ip="127.0.0.1", client_ip=127.0.0.1, tenant_name=sys, tenant_id=1, user_name=ocp_monitor, host_name=%, sessid=3225503935, proxy_sessid=0, sess_create_time=0, from_proxy=false, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=762509, proxy_capability=0, use_ssl=false, c/s protocol="OB_MYSQL_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:03.025447] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56168-0-0] [lt=8] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11007, add_timestamp:1723774803025431}) [2024-08-16 10:20:03.025480] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676703-0-0] [lt=13] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11007, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:03.025941] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:224) [31439][pnio1][T0][YB42C0A80080-00061CCCC1B56168-0-0] [lt=26] gts local cache not updated(result={tenant_id:1, status:0, srr:[mts=1723774803025531], gts_start:1723774803025651001, gts_end:1723774803025651001}) [2024-08-16 10:20:03.026034] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676703-0-0] [lt=23] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11007, locations=[{table_id:11007, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:03.026131] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676703-0-0] [lt=95] success to process renew task(task={tenant_id:1, table_id:11007, add_timestamp:1723774803025431}, locations=[{table_id:11007, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}, {table_id:11007, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774803026008, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:03.027525] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C1B-0-0] [lt=56] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774803027489) [2024-08-16 10:20:03.029575] INFO [SERVER] fill_tenant_ids_ (ob_all_virtual_tx_stat.cpp:139) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEB-0-0] [lt=5] succeed to get tenant ids(all_tenants_=[1, 1001, 1002]) [2024-08-16 10:20:03.031497] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56168-0-0] [lt=27][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=3, len=45, pos=34, bitmap=NULL) [2024-08-16 10:20:03.032127] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56168-0-0] [lt=22][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=3, len=61, pos=38, bitmap=NULL) [2024-08-16 10:20:03.032649] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C7A-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=7, len=401, pos=62, bitmap=NULL) [2024-08-16 10:20:03.032722] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_event.cpp:102) [4765][T1_L0_G0][T1][YB42C0A80080-00061CCCBC524C7A-0-0] [lt=25][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:03.032809] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56168-0-0] [lt=19][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=4, len=69, pos=40, bitmap=NULL) [2024-08-16 10:20:03.034653] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=36] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=0) [2024-08-16 10:20:03.036830] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa71f1f4030, session_id:3225571382, trace_id:YB42C0A80080-00061CCCC85CFAEC-0-0, sql_handling_stage:3, sql_initiative_shutdown:false, reader:{fd:224}, err:5, last_decode_time:1723774803035960, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:1747, pending_flag:1, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:03.036869] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=30] kill and revert session(conn.sessid_=3225571382, proxy_sessid=0, server_id=16, ret=0) [2024-08-16 10:20:03.036860] WDIAG [SERVER] disconnect (obmp_packet_sender.cpp:830) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEC-0-0] [lt=27][errcode=0] server close connection(sessid=3225571382, proxy_sessid=0, stack="0x12a0688c 0xa219970 0xa1e504b 0xa527c01 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:03.036887] INFO [SERVER] process (obmp_quit.h:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEC-0-0] [lt=24] quit [2024-08-16 10:20:03.036896] WDIAG [RPC.OBMYSQL] async_write_data (ob_sql_sock_session.cpp:163) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEC-0-0] [lt=7][errcode=-4009] sock has error(ret=-4009) [2024-08-16 10:20:03.036918] WDIAG [SERVER] flush_buffer (obmp_packet_sender.cpp:1032) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEC-0-0] [lt=20][errcode=-4009] write response fail(ret=-4009) [2024-08-16 10:20:03.036926] WDIAG [SERVER] response (obmp_base.cpp:87) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEC-0-0] [lt=7][errcode=-4009] failed to flush_buffer(ret=-4009) [2024-08-16 10:20:03.036933] WDIAG [RPC.FRAME] run (ob_sql_processor.cpp:48) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEC-0-0] [lt=5][errcode=-4009] response rpc result fail(ret=-4009) [2024-08-16 10:20:03.037683] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa71f1dc030, session_id:3225599215, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:false, reader:{fd:129}, err:5, last_decode_time:1723774803025147, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:447, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:03.037727] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=35] kill and revert session(conn.sessid_=3225599215, proxy_sessid=0, server_id=16, ret=0) [2024-08-16 10:20:03.037739] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=10] can close safely, do destroy(*s={this:0x7fa71f1f4030, session_id:3225571382, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:true, reader:{fd:224}, err:5, last_decode_time:1723774803035960, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:1752, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-08-16 10:20:03.037765] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=10] unlock session of tenant(conn.sessid_=3225571382, proxy_sessid=0, conn.tenant_id_=1) [2024-08-16 10:20:03.037795] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=7][errcode=0] uref leak check(node->uref_=1074538686, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:03.037776] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=9] connection close(sessid=3225571382, proxy_sessid=0, tenant_id=1, server_id=16, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCC85CFAEB-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:22, pkt_rec[0]:{send:obp_mysql_header_{len_:11, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{receive:obp_mysql_header_{len_:254, rec_:254, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[2]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{send:obp_mysql_header_{pkt_num_:10, seq_:12}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[4]:{send:obp_mysql_header_{len_:9, seq_:13}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_mysql_header_{pkt_num_:710, seq_:208}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[6]:{send:obp_mysql_header_{len_:9, seq_:209}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[7]:{receive:obp_mysql_header_{len_:318, rec_:318, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[8]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[9]:{send:obp_mysql_header_{pkt_num_:1, seq_:3}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[10]:{send:obp_mysql_header_{len_:9, seq_:5}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[11]:{receive:obp_mysql_header_{len_:693, rec_:693, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[12]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[13]:{send:obp_mysql_header_{pkt_num_:3, seq_:5}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[14]:{send:obp_mysql_header_{len_:9, seq_:7}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[15]:{receive:obp_mysql_header_{len_:365, rec_:365, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[16]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[17]:{send:obp_mysql_header_{pkt_num_:1, seq_:3}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[18]:{send:obp_mysql_header_{len_:9, seq_:4}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[19]:{send:obp_mysql_header_{pkt_num_:1, seq_:5}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[20]:{send:obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[21]:{receive:obp_mysql_header_{len_:1, rec_:1, seq_:0}, pkt_name:"COM_QUIT", obp_mysql_header_.type_:1, obp_mysql_header_.is_send_:0}], disconnect_state=3) [2024-08-16 10:20:03.037848] INFO [SERVER] run (obmp_disconnect.cpp:82) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=17] free session successfully(sessid=3225571382) [2024-08-16 10:20:03.037859] INFO [SERVER] run (obmp_disconnect.cpp:89) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=9] mark session id unused(sessid=3225571382) [2024-08-16 10:20:03.037855] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=74] sock ref clean, do destroy(*s={this:0x7fa71f1dc030, session_id:3225599215, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:false, reader:{fd:129}, err:5, last_decode_time:1723774803025147, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:447, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:03.037890] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=17] unlock session of tenant(conn.sessid_=3225599215, proxy_sessid=0, conn.tenant_id_=1) [2024-08-16 10:20:03.037909] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=3][errcode=0] uref leak check(node->uref_=1074538687, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:03.037908] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=16] connection close(sessid=3225599215, proxy_sessid=0, tenant_id=1, server_id=16, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCBC524C7A-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:7, pkt_rec[0]:{send:obp_mysql_header_{len_:11, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{receive:obp_mysql_header_{len_:342, rec_:342, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[2]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{send:obp_mysql_header_{pkt_num_:8, seq_:10}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[4]:{send:obp_mysql_header_{len_:9, seq_:11}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_mysql_header_{pkt_num_:164, seq_:174}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[6]:{send:obp_mysql_header_{len_:9, seq_:175}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}], disconnect_state=4) [2024-08-16 10:20:03.037944] INFO [SERVER] run (obmp_disconnect.cpp:82) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=8] free session successfully(sessid=3225599215) [2024-08-16 10:20:03.037953] INFO [SERVER] run (obmp_disconnect.cpp:89) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=8] mark session id unused(sessid=3225599215) [2024-08-16 10:20:03.045714] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:03.045814] INFO [CLOG] push_log_replay_task (ob_replay_status.cpp:1140) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=8] submit pre barrier log success(log_type=17, ls_id={id:1262}, lsn={lsn:8394847317}, scn={val:1723774802760070002, v:0}, is_pre_barrier=true, is_post_barrier=false, log_size=24) [2024-08-16 10:20:03.045841] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=37] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:03.045866] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1220) [2024-08-16 10:20:03.045881] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:03.045795] INFO [CLOG] do_replay_task_ (ob_log_replay_service.cpp:901) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=37] wait for barrier(ret=-4023, replay_status={ls_id_:{id:1262}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:4, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774803045466, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10679383, ret_code_:0}}, next_to_submit_lsn_:{lsn:8394847261}, next_to_submit_scn_:{val:1723774802760070002, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516551920929002, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69ec05000, next_round_pread_size:3678208, curr_read_pos:56, curr_read_buf_start_pos:0, curr_read_buf_end_pos:112, log_storage_:{IteratorStorage:{start_lsn:{lsn:8394847261}, end_lsn:{lsn:8394847373}, read_buf:{buf_len_:3682304, buf_:0x7fa69ec05000}, block_size:67104768, log_storage_:0x7fa6be533470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774802760070002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:24, scn_:{val:1723774802760070002, v:0}, data_checksum:3214331258, flag:1}}, init_mode_version:0, accumulate_checksum:772067526, curr_entry_is_padding:0, padding_entry_size:56, padding_entry_scn:{val:1723774802760070002, v:0}}}}}, replay_task={ls_id_:{id:1262}, log_type:17, log_type:"RESERVED_SNAPSHOT", lsn_:{lsn:8394847317}, scn_:{val:1723774802760070002, v:0}, is_pre_barrier_:true, is_post_barrier_:false, read_log_size_:24, replay_hint_:1723774803041849, first_handle_ts_:1723774803045791, replay_cost_:-1, retry_cost_:-1, read_log_buf_:0x7fa5a417eb10, decompression_buf_:null, has_decompressed_:false, decompressed_log_size_:0, base_header_len_:16}) [2024-08-16 10:20:03.045888] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:03.045893] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:03.046033] INFO [STORAGE] inner_update_reserved_snapshot (ob_ls_reserved_snapshot_mgr.cpp:288) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=12] success to update reserved snapshot(ret=0, ls_id={id:1262}, min_reserved_snapshot_=1723773000638406000) [2024-08-16 10:20:03.046062] INFO [CLOG] do_replay_task_ (ob_log_replay_service.cpp:919) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=24] pre barrier log replay succ(replay_task={ls_id_:{id:1262}, log_type:17, log_type:"RESERVED_SNAPSHOT", lsn_:{lsn:8394847317}, scn_:{val:1723774802760070002, v:0}, is_pre_barrier_:true, is_post_barrier_:false, read_log_size_:24, replay_hint_:1723774803041849, first_handle_ts_:1723774803046010, replay_cost_:48, retry_cost_:-1, read_log_buf_:0x7fa5a417eb20, decompression_buf_:null, has_decompressed_:false, decompressed_log_size_:0, base_header_len_:16}, replay_status={ls_id_:{id:1262}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774803045466, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10679383, ret_code_:0}}, next_to_submit_lsn_:{lsn:8394847373}, next_to_submit_scn_:{val:1723774802760070003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516551920929002, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69ec05000, next_round_pread_size:3678208, curr_read_pos:112, curr_read_buf_start_pos:0, curr_read_buf_end_pos:112, log_storage_:{IteratorStorage:{start_lsn:{lsn:8394847261}, end_lsn:{lsn:8394847373}, read_buf:{buf_len_:3682304, buf_:0x7fa69ec05000}, block_size:67104768, log_storage_:0x7fa6be533470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774802760070002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:24, scn_:{val:1723774802760070002, v:0}, data_checksum:3214331258, flag:1}}, init_mode_version:0, accumulate_checksum:772067526, curr_entry_is_padding:0, padding_entry_size:56, padding_entry_scn:{val:1723774802760070002, v:0}}}}}, replay_queue_idx=25, ret=0) [2024-08-16 10:20:03.046409] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=23] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=801652736, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:03.046691] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=32] Wash time detail, (compute_wash_size_time=125, refresh_score_time=81, wash_time=16) [2024-08-16 10:20:03.047750] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:03.050501] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Cache wash clean map node details(ret=0, clean_node_count=87, clean_time=0, clean_start_pos=2453646, clean_num=62914) [2024-08-16 10:20:03.053263] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.053285] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:03.053297] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.053307] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.053321] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.054500] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=23] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:03.056357] INFO [CLOG] handle_submit_task_ (ob_log_replay_service.cpp:1208) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=23] succ to submit log task to replay service(to_submit_lsn={lsn:8394847429}, to_submit_scn={val:1723774802760070003, v:0}, replay_status={ls_id_:{id:1262}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774803056262, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10679383, ret_code_:0}}, next_to_submit_lsn_:{lsn:8394847495}, next_to_submit_scn_:{val:1723774802760070004, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516551920929002, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69ec05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8394847373}, end_lsn:{lsn:8394847495}, read_buf:{buf_len_:3682304, buf_:0x7fa69ec05000}, block_size:67104768, log_storage_:0x7fa6be533470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774802760070003, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774802760070003, v:0}, data_checksum:2892311396, flag:0}}, init_mode_version:0, accumulate_checksum:637356650, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774802760070003, v:0}}}}}) [2024-08-16 10:20:03.065152] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:03.065184] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=31][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:03.078025] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=28] PNIO [ratelimit] time: 1723774803078021, bytes: 1496436960180, bw: 0.140831 MB/s, add_ts: 1000447, add_bytes: 147738 [2024-08-16 10:20:03.078800] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8622-0-0] [lt=23] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:03.079624] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=55757746, this={dio_aligned_buf:{buf_write_offset:3116, buf_padding_size:980, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:47790933, truncate_used_ts:11339185}, log_block_size:67108864, dir_fd:200, io_fd:106}, aligned_buf_len=4096, aligned_buf="LH", aligned_block_offset=55754752, buf_len=122, total_write_size_=13066, total_write_size_after_dio_=421888, ob_pwrite_used_ts=57978, count=100, each_pwrite_cost=579) [2024-08-16 10:20:03.079766] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=23] PNIO [ratelimit] time: 1723774803079765, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007634, add_bytes: 0 [2024-08-16 10:20:03.080273] INFO [LIB] stat (utility.h:1197) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=30] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=62, stat_interval=1000000, avg (count/cost)=139, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:03.080297] INFO [LIB] stat (utility.h:1197) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=23] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=62, stat_interval=1000000, avg (count/cost)=7, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:03.093231] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF105-0-0] [lt=24][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509880, log_task={header:{begin_lsn:{lsn:18544225358}, end_lsn:{lsn:18544225480}, log_id:53509880, min_scn:{val:1723774802883195001, v:0}, max_scn:{val:1723774802883195001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544225236}, prev_proposal_id:7, committed_end_lsn:{lsn:18544225358}, data_checksum:1913811101, accum_checksum:716727021, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774802993058, freeze_ts:1723774802993058, submit_ts:1723774802993097, flushed_ts:1723774802993873, gen_to_freeze cost time:0, gen_to_submit cost time:39, submit_to_flush cost time:776}, fs_cb_begin_ts=1723774803093212, log_life_time=100154) [2024-08-16 10:20:03.093921] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=48, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:20:03.096261] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [2785][T1001_ObTimer][T1001][Y0-0000000000000000-0-0] [lt=11] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:20:03.096300] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:230) [2785][T1001_ObTimer][T1001][Y0-0000000000000000-0-0] [lt=25] [tabletchange] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12275) [2024-08-16 10:20:03.102179] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.102208] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=26] handle task succ(id={id:1260}) [2024-08-16 10:20:03.102225] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.102240] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.102265] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.116057] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80085-00061CCD1E636C13-0-0] [lt=42][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:03.134788] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2747][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=8] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:20:03.134899] INFO [STORAGE] update_info (ob_tenant_freeze_info_mgr.cpp:766) [2747][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=30] ObTenantFreezeInfoMgr success to update infos(snapshot_gc_ts=1723774800639482000, min_major_snapshot=1, cur_idx=0, info_list_[0]=[{freeze_version:1, schema_version:1, cluster_version:17180000768}, {freeze_version:1717696802500718000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1717783200446161000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1717869603412972000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1717956001367405000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718042404278073000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718128802164565000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718215200128515001, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718301603134619000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718388001691923000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718474404651300000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718560802433273000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718647201110312000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718733600101602000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718820003144173000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718906401119406000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718992804061366000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719079201877087000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719165604743651000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719252002784390000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719338400977208000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719424804091498000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719511202206653001, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719597600247948000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719684003120658000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719770401089217000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719856804103163000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719943202295808000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1720029600426916000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1720116003503755000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1720202401636328000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1720288804751256000, schema_version:1713774538068224, cluster_version:17180000768}], info_list_[1]=[{freeze_version:1, schema_version:1, cluster_version:17180000768}, {freeze_version:1717696802500718000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1717783200446161000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1717869603412972000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1717956001367405000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718042404278073000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718128802164565000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718215200128515001, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718301603134619000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718388001691923000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718474404651300000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718560802433273000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718647201110312000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718733600101602000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718820003144173000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718906401119406000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1718992804061366000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719079201877087000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719165604743651000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719252002784390000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719338400977208000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719424804091498000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719511202206653001, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719597600247948000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719684003120658000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719770401089217000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719856804103163000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1719943202295808000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1720029600426916000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1720116003503755000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1720202401636328000, schema_version:1713774538068224, cluster_version:17180000768}, {freeze_version:1720288804751256000, schema_version:1713774538068224, cluster_version:17180000768}], snapshots_[0]=[], snapshots_[1]=[]) [2024-08-16 10:20:03.136584] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DD-0-0] [lt=72][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:03.137245] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DD-0-0] [lt=22] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:03.145785] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:03.145926] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=31] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:03.145946] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:03.145954] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:03.146538] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:03.147831] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:134) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20] ObStandbyTimestampService state(*this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:20:03.147868] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=35][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:03.153340] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.153362] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:03.153373] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.153382] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.153396] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.155220] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AED-0-0] [lt=53] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1332, avg_group_log_size=121) [2024-08-16 10:20:03.155255] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AED-0-0] [lt=31] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1332) [2024-08-16 10:20:03.155769] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=40] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1332) [2024-08-16 10:20:03.156327] INFO [PALF] read (log_cache.cpp:100) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=89] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1332, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:03.172227] INFO [CLOG] runTimerTask (ob_log_replay_service.cpp:159) [2760][T1001_ReplayPro][T1001][Y0-0000000000000000-0-0] [lt=40] dump tenant replay process(tenant_id=1001, unreplayed_log_size(MB)=0, estimate_time(second)=0, replayed_log_size(MB)=0, last_replayed_log_size(MB)=0, round_cost_time(second)=10, pending_replay_log_size(MB)=0) [2024-08-16 10:20:03.183721] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=25] gc stale ls task succ [2024-08-16 10:20:03.187123] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAED-0-0] [lt=3][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=2, len=64, pos=51, bitmap=NULL) [2024-08-16 10:20:03.187558] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAED-0-0] [lt=26][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=25, len=250, pos=221, bitmap=NULL) [2024-08-16 10:20:03.193388] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF10B-0-0] [lt=12][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509881, log_task={header:{begin_lsn:{lsn:18544225480}, end_lsn:{lsn:18544225602}, log_id:53509881, min_scn:{val:1723774802883195002, v:0}, max_scn:{val:1723774802883195002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544225358}, prev_proposal_id:7, committed_end_lsn:{lsn:18544225480}, data_checksum:1913811101, accum_checksum:2152275266, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774803093178, freeze_ts:1723774803093178, submit_ts:1723774803093190, flushed_ts:1723774803093884, gen_to_freeze cost time:0, gen_to_submit cost time:12, submit_to_flush cost time:694}, fs_cb_begin_ts=1723774803193362, log_life_time=100184) [2024-08-16 10:20:03.198046] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [2549][T1002_LCLSender][T1002][Y0-0000000000000000-0-0] [lt=45] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=1135, total_destructed_detector=1135, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x7fa7110382b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-08-16 10:20:03.202241] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.202272] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=28] handle task succ(id={id:1260}) [2024-08-16 10:20:03.202281] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.202290] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.202303] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.202485] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AB0-0-0] [lt=20] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:03.202532] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AB0-0-0] [lt=33] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774802760070004, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774802760070004, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:03.213369] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [2786][T1001_TabletShe][T1001][Y0-0000000000000000-0-0] [lt=13] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:20:03.213409] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:101) [2786][T1001_TabletShe][T1001][Y0-0000000000000000-0-0] [lt=24] [emptytablet] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12275) [2024-08-16 10:20:03.231029] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2752][T1001_TenantMet][T1001][Y0-0000000000000000-0-0] [lt=49] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=0, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:03.231836] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007E-00061EFF757F6EAE-0-0] [lt=45][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:03.236300] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=21] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=2673845, replace_num=31457) [2024-08-16 10:20:03.236326] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=23] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=10) [2024-08-16 10:20:03.236452] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:03.236562] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=16] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:03.236591] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=9] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:03.236911] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=19] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:03.236929] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=13] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:03.237089] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=17] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:03.237102] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:03.245866] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:03.245998] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=33] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:03.246015] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:03.246022] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:03.247904] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=16][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:03.250948] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80086-00061EFF757FAF13-0-0] [lt=17][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:03.251036] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:03.251354] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=57] Wash time detail, (compute_wash_size_time=151, refresh_score_time=68, wash_time=16) [2024-08-16 10:20:03.253421] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.253443] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:03.253453] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.253462] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.253476] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.254876] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=15] Cache wash clean map node details(ret=0, clean_node_count=109, clean_time=0, clean_start_pos=2516560, clean_num=62914) [2024-08-16 10:20:03.278555] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8626-0-0] [lt=51] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:03.279342] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=50] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=717, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:03.279378] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:03.279411] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=31, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:03.279473] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=39, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:03.279768] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE631-0-0] [lt=27] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=16, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:03.279792] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE631-0-0] [lt=22] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=11096, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:03.293596] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF111-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1501, avg_group_log_size=150) [2024-08-16 10:20:03.293656] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF111-0-0] [lt=37][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509882, log_task={header:{begin_lsn:{lsn:18544225602}, end_lsn:{lsn:18544225724}, log_id:53509882, min_scn:{val:1723774802883195003, v:0}, max_scn:{val:1723774802883195003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544225480}, prev_proposal_id:7, committed_end_lsn:{lsn:18544225602}, data_checksum:1913811101, accum_checksum:4057909647, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774803193330, freeze_ts:1723774803193331, submit_ts:1723774803193343, flushed_ts:1723774803194238, gen_to_freeze cost time:1, gen_to_submit cost time:13, submit_to_flush cost time:895}, fs_cb_begin_ts=1723774803293642, log_life_time=100312) [2024-08-16 10:20:03.300157] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=15] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:20:03.300184] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=24] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=1, size_used=5170, mem_used=39515136) [2024-08-16 10:20:03.302323] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.302362] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=37] handle task succ(id={id:1260}) [2024-08-16 10:20:03.302372] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.302382] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.302408] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.310643] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE82-0-0] [lt=41] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:03.310702] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE82-0-0] [lt=28] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774802864858004, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774802864858004, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:03.340564] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=32] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574636, offset=42146341}, end_cursor:ObLogCursor{file_id=65, log_id=18574637, offset=42146582}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751700050, len:1499, log_data_len:241, seq:18574636, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:03.341260] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=46] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574637, offset=42146582}, end_cursor:ObLogCursor{file_id=65, log_id=18574638, offset=42146831}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751664050, len:1258, log_data_len:249, seq:18574637, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:03.342870] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA5-0-0] [lt=31] push gts task success(*task={:""}) [2024-08-16 10:20:03.345957] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:03.346106] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:03.346131] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:03.346142] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:03.346688] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=693, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:03.346719] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=40, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:03.346750] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=17] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=18, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:03.347481] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE633-0-0] [lt=39] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:03.347499] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE633-0-0] [lt=17] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=1340, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:03.347939] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:03.350966] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA5-0-0] [lt=22] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:03.353503] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.353524] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:03.353536] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.353547] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.353563] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.353660] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=34] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574638, offset=42146831}, end_cursor:ObLogCursor{file_id=65, log_id=18574639, offset=42147080}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751650050, len:1009, log_data_len:249, seq:18574638, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:03.354286] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=26] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574639, offset=42147080}, end_cursor:ObLogCursor{file_id=65, log_id=18574640, offset=42147329}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516e0050, len:760, log_data_len:249, seq:18574639, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:03.378727] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8628-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:03.384371] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=65] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:03.393734] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF117-0-0] [lt=37] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1501) [2024-08-16 10:20:03.393787] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF117-0-0] [lt=28] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=21, avg_log_submit_to_flush_time=951, avg_log_submit_to_slide_time=100141) [2024-08-16 10:20:03.393876] INFO [PALF] read (log_cache.cpp:100) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1501, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:03.402399] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=198] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.402427] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:20:03.402438] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.402449] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.402464] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.425544] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=22] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:03.425662] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=19] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:03.425941] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=45] gts statistics(tenant_id=1, gts_rpc_cnt=16, get_gts_cache_cnt=104, get_gts_with_stc_cnt=6, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-08-16 10:20:03.436431] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=9] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=9) [2024-08-16 10:20:03.446032] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:03.446184] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:03.446201] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:03.446208] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:03.447982] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=27][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:03.453586] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.453609] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:03.453621] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.453632] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.453647] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.455348] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:03.455615] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=28] Wash time detail, (compute_wash_size_time=123, refresh_score_time=69, wash_time=15) [2024-08-16 10:20:03.456008] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=20] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:03.456032] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=23] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:03.458019] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=11] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:03.458062] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=20] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:03.458110] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=41] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:03.458121] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:03.458149] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=16] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:03.458172] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=18] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:03.458212] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=5] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:03.458225] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=12] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:03.458248] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:03.458266] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAEE-0-0] [lt=17] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:03.458937] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=44] Cache wash clean map node details(ret=0, clean_node_count=96, clean_time=0, clean_start_pos=2579474, clean_num=62914) [2024-08-16 10:20:03.472625] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [31766][T1_ObXAHbWorker][T1][Y0-0000000000000000-0-0] [lt=9] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-08-16 10:20:03.489223] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7DF-0-0] [lt=25] [RS_MGR] force set rs list(master_rs="192.168.0.126:2882") [2024-08-16 10:20:03.489284] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7DF-0-0] [lt=39] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"192.168.0.128:2882", server_id:16, rs_addr:"192.168.0.126:2882", rs_server_status:1, epoch_id:11}, hb_response={zone:"zone2", server:"192.168.0.128:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1720514727760575, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=64) [2024-08-16 10:20:03.502489] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.502526] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=34] handle task succ(id={id:1260}) [2024-08-16 10:20:03.502539] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.502550] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.502566] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.526122] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=29] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=6963, mem_used=12478464) [2024-08-16 10:20:03.529765] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1961) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(dag_cnt=0, map_size=0) [2024-08-16 10:20:03.529799] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1971) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=32] dump_dag_status(running_dag_net_map_size=0, blocking_dag_net_list_size=0) [2024-08-16 10:20:03.529809] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(priority="PRIO_COMPACTION_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:03.529819] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(priority="PRIO_HA_HIGH", low_limit=8, up_limit=8, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:03.529823] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_COMPACTION_MID", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:03.529827] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_HA_MID", low_limit=5, up_limit=5, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:03.529831] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_COMPACTION_LOW", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:03.529837] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(priority="PRIO_HA_LOW", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:03.529847] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(priority="PRIO_DDL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:03.529852] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(priority="PRIO_DDL_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:03.529856] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_TTL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:03.529860] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:03.529881] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=20] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:03.529888] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:03.529893] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:03.529897] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:03.529901] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:03.529907] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:03.529915] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:03.529920] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:03.529924] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:03.529929] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:03.529938] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:03.529942] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:03.529947] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:03.529950] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:03.529965] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=14] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:03.529972] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:03.529977] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:03.529981] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:03.529986] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:03.529995] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530000] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530004] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530008] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530012] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530016] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530034] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=18] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530041] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530046] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530050] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530054] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530058] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530063] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530093] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=29] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530100] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530105] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530109] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530115] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530126] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=11] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530131] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530135] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530139] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530143] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530147] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530150] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530154] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530177] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=23] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530184] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530189] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:03.530193] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530198] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-08-16 10:20:03.530209] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-08-16 10:20:03.530214] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-08-16 10:20:03.530218] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-08-16 10:20:03.530223] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:03.530240] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=16] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:03.530246] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:03.530251] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:03.530257] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:03.530268] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:03.530273] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:03.530278] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:03.530291] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=12] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:03.530297] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:03.530302] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:03.530307] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:03.530319] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=11] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:03.530324] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:03.530327] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:03.530331] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:03.530335] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:03.530339] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:03.530344] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:03.530354] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530359] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:03.530363] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530367] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:03.530371] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530376] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:03.530390] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=13] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530396] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:03.530401] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530406] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:03.530416] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530421] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:03.530425] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530429] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:03.530432] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530436] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:03.530440] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530444] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:03.530448] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:03.530451] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, dag_count=0) [2024-08-16 10:20:03.530455] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, scheduled_task_count=0) [2024-08-16 10:20:03.530459] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, dag_count=0) [2024-08-16 10:20:03.530463] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, scheduled_task_count=0) [2024-08-16 10:20:03.530467] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, dag_count=0) [2024-08-16 10:20:03.530471] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-08-16 10:20:03.530474] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, dag_count=0) [2024-08-16 10:20:03.530478] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-08-16 10:20:03.530482] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, dag_count=0) [2024-08-16 10:20:03.530486] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, scheduled_task_count=0) [2024-08-16 10:20:03.530490] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status[DAG_NET](type="DAG_NET_MIGRATION", dag_net_count=0) [2024-08-16 10:20:03.530496] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status[DAG_NET](type="DAG_NET_PREPARE_MIGRATION", dag_net_count=0) [2024-08-16 10:20:03.530507] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=10] dump_dag_status[DAG_NET](type="DAG_NET_COMPLETE_MIGRATION", dag_net_count=0) [2024-08-16 10:20:03.530511] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER", dag_net_count=0) [2024-08-16 10:20:03.530514] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_BACKUP", dag_net_count=0) [2024-08-16 10:20:03.530517] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_RESTORE", dag_net_count=0) [2024-08-16 10:20:03.530522] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status[DAG_NET](type="DAG_NET_TYPE_BACKUP_CLEAN", dag_net_count=0) [2024-08-16 10:20:03.530527] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER_BACKFILL_TX", dag_net_count=0) [2024-08-16 10:20:03.530541] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1991) [2746][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=13] dump_dag_status(total_worker_cnt=43, total_running_task_cnt=0, work_thread_num=43, scheduled_task_cnt=0) [2024-08-16 10:20:03.533530] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=34] start do ls ha handler(ls_id_array_=[{id:1079}, {id:1260}, {id:1262}, {id:1261}]) [2024-08-16 10:20:03.535572] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [31651][T1_PalfGC][T1][Y0-0000000000000000-0-0] [lt=46] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}) [2024-08-16 10:20:03.539704] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [31763][T1_LCLSender][T1][Y0-0000000000000000-0-0] [lt=64] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=1135, total_destructed_detector=1135, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x7fa76bdfe2b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-08-16 10:20:03.541276] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=9] trace max work area(auto_calc=true, tenant_max_memory_limit=1073741824, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=53687091, max_wa_memory_size=53687091, tmp_max_wa_memory_size=53687091, pre_mem_target=53687091, remain_memory_size=53687091, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=332681216, washable_size=-2, max_workarea_memory_size=53687091, max_tenant_memory_size=741060608, wash_ratio=6, tenant_id=1001) [2024-08-16 10:20:03.541376] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=57] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=6710886, calc_global_bound_size=1048576000000, mem_target_=53687091, wa_max_memory_size=53687091, profile_cnt_=0, total_memory_size=0, max_wa_size=53687091, avg_bound_size=6710886, max_bound_size=6710886, min_bound_size_=1048576) [2024-08-16 10:20:03.541405] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=26] timer to calc global bound size(ret=0, global_bound_size_=6710886, manual_calc_cnt_=0, drift_size_=0, pre_drift_size=0, wa_max_memory_size=53687091, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1001, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=6710886, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=53687091, auto_calc=true, sql_mem_callback_.get_total_dump_size()=0) [2024-08-16 10:20:03.541455] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [31638][T1_Occam][T1][YB42C0A80080-00061CCCBF6D3E0C-0-0] [lt=22] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:03.541455] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=14] trace max work area(auto_calc=true, tenant_max_memory_limit=6442450944, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=3865470566, max_wa_memory_size=3865470566, tmp_max_wa_memory_size=3865470566, pre_mem_target=3865470566, remain_memory_size=3865470566, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=2461601792, washable_size=-2, max_workarea_memory_size=3865470566, max_tenant_memory_size=3980849152, wash_ratio=6, tenant_id=1002) [2024-08-16 10:20:03.541524] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=17] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=483183820, calc_global_bound_size=1048576000000, mem_target_=3865470566, wa_max_memory_size=3865470566, profile_cnt_=0, total_memory_size=0, max_wa_size=3865470566, avg_bound_size=483183820, max_bound_size=483183820, min_bound_size_=1048576) [2024-08-16 10:20:03.541550] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=24] timer to calc global bound size(ret=0, global_bound_size_=483183820, manual_calc_cnt_=19637, drift_size_=0, pre_drift_size=0, wa_max_memory_size=3865470566, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1002, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=483183820, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=3865470566, auto_calc=true, sql_mem_callback_.get_total_dump_size()=1112786688) [2024-08-16 10:20:03.546115] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:03.546225] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:03.546257] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:03.546271] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:03.548054] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:03.553660] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.553682] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:03.553693] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.553704] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.553722] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.554857] INFO [STORAGE.TRANS] amend (ob_tenant_weak_read_server_version_mgr.cpp:192) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=27] [WRS] tenant has no valid partition on current server, generate weak read snapshot version based on current timestamp(tenant_id=1001, epoch_tstamp_=1723774803554839, total_part_count_=0, valid_inner_part_count_=0, valid_user_part_count_=0, version_={val:1723774798924916000, v:0}, base_version_when_no_valid_partition={val:1723774798924916000, v:0}) [2024-08-16 10:20:03.555368] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AF5-0-0] [lt=18] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:03.555413] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AF5-0-0] [lt=20][errcode=-4389] log_task life cost too much time(palf_id=1262, self="192.168.0.128:2882", log_id=34459379, log_task={header:{begin_lsn:{lsn:8394847861}, end_lsn:{lsn:8394847983}, log_id:34459379, min_scn:{val:1723774803260345001, v:0}, max_scn:{val:1723774803260345001, v:0}, data_len:66, proposal_id:2, prev_lsn:{lsn:8394847739}, prev_proposal_id:2, committed_end_lsn:{lsn:8394847861}, data_checksum:2071192402, accum_checksum:2954088774, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774803455394, freeze_ts:1723774803455394, submit_ts:1723774803455405, flushed_ts:1723774803459241, gen_to_freeze cost time:0, gen_to_submit cost time:11, submit_to_flush cost time:3836}, fs_cb_begin_ts=1723774803555402, log_life_time=100008) [2024-08-16 10:20:03.566144] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 406.00) [2024-08-16 10:20:03.566167] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=18] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:03.566245] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=30] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.99, rt: 558.75) [2024-08-16 10:20:03.566260] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=12] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SHARED_BLOCK_ReaderWriter_IO, mode: read, size: 4096.00, iops: 281.99, rt: 226.14) [2024-08-16 10:20:03.566269] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:03.566292] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=18] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566301] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566327] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=24] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566335] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566342] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566347] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566351] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566357] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566366] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566371] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566380] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566385] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566389] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566395] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566405] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.566421] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=14] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:03.584654] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=11] ====== check clog disk timer task ====== [2024-08-16 10:20:03.584681] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=24] get_disk_usage(ret=0, capacity(MB):=2150, used(MB):=0) [2024-08-16 10:20:03.584697] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=10] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=676457349, need_update_checkpoint_scn=false) [2024-08-16 10:20:03.585742] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=13] ====== check clog disk timer task ====== [2024-08-16 10:20:03.585767] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=22] get_disk_usage(ret=0, capacity(MB):=0, used(MB):=0) [2024-08-16 10:20:03.585782] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=10] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=0, need_update_checkpoint_scn=false) [2024-08-16 10:20:03.591345] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=20][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:03.593718] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=31] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:03.593694] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF123-0-0] [lt=34][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509885, log_task={header:{begin_lsn:{lsn:18544225968}, end_lsn:{lsn:18544226090}, log_id:53509885, min_scn:{val:1723774803384133001, v:0}, max_scn:{val:1723774803384133001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544225846}, prev_proposal_id:7, committed_end_lsn:{lsn:18544225968}, data_checksum:1913811101, accum_checksum:795412247, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774803493598, freeze_ts:1723774803493598, submit_ts:1723774803493607, flushed_ts:1723774803494441, gen_to_freeze cost time:0, gen_to_submit cost time:9, submit_to_flush cost time:834}, fs_cb_begin_ts=1723774803593680, log_life_time=100082) [2024-08-16 10:20:03.593810] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=24][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:03.593833] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=21][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:03.593843] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=9] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:03.594376] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=22, stat_interval=1000000, avg (count/cost)=36, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:03.602560] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.602583] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:03.602595] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.602605] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.602634] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.607362] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=22] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:03.617699] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=27] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60688, mem_used=137263104) [2024-08-16 10:20:03.636561] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=49] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=8) [2024-08-16 10:20:03.645271] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=30] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:03.646168] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:03.646296] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=30] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:03.646314] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:03.646321] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:03.647471] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73609-0-0] [lt=61] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=6, avg_log_gen_to_submit_time=71, avg_log_submit_to_flush_time=763, avg_log_submit_to_slide_time=1270) [2024-08-16 10:20:03.648142] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=43][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:03.653761] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.653792] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=28] handle task succ(id={id:1260}) [2024-08-16 10:20:03.653806] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.653831] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=23] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.653858] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.656061] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=889, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:03.656196] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=38, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:03.656531] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7360A-0-0] [lt=31] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=15, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:03.656565] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7360A-0-0] [lt=33] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=10201, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:03.659470] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:03.659748] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=117, refresh_score_time=79, wash_time=16) [2024-08-16 10:20:03.663032] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Cache wash clean map node details(ret=0, clean_node_count=96, clean_time=0, clean_start_pos=2642388, clean_num=62914) [2024-08-16 10:20:03.702648] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.702676] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=26] handle task succ(id={id:1260}) [2024-08-16 10:20:03.702689] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.702699] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.702714] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.746241] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:03.746350] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=48, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:03.746388] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:03.746401] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:03.746409] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:03.746413] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:03.748217] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=29][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:03.753849] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.753870] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:03.753881] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.753892] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.753907] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.768126] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=21] ====== tenant freeze timer task ====== [2024-08-16 10:20:03.768160] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=20][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:03.793592] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF12F-0-0] [lt=9] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:03.793656] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF12F-0-0] [lt=34] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:03.793664] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF12F-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=91014, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:03.793671] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF12F-0-0] [lt=4][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509887, log_task={header:{begin_lsn:{lsn:18544226212}, end_lsn:{lsn:18544226334}, log_id:53509887, min_scn:{val:1723774803384133003, v:0}, max_scn:{val:1723774803384133003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544226090}, prev_proposal_id:7, committed_end_lsn:{lsn:18544226212}, data_checksum:1913811101, accum_checksum:3402271196, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774803693585, freeze_ts:1723774803693585, submit_ts:1723774803693593, flushed_ts:1723774803694372, gen_to_freeze cost time:0, gen_to_submit cost time:8, submit_to_flush cost time:779}, fs_cb_begin_ts=1723774803793644, log_life_time=100059) [2024-08-16 10:20:03.794201] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=53] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1623) [2024-08-16 10:20:03.802729] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.802751] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:03.802763] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.802773] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.802785] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.833766] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=13] new mysql sessid created(conn.sessid_=3225555614, support_ssl=false) [2024-08-16 10:20:03.833820] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=30] sm conn init succ(conn.sessid_=3225555614, sess.client_addr_="192.168.0.127:39224") [2024-08-16 10:20:03.833836] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=10] accept one succ(*s={this:0x7fa70d9fed70, session_id:3225555614, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:129}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:03.834057] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31556][sql_nio1][T1][Y0-0000000000000000-0-0] [lt=26] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:03.834758] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D31-0-0] [lt=56][errcode=0] uref leak check(node->uref_=1074538688, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:03.835040] INFO [SERVER] process (obmp_connect.cpp:475) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D31-0-0] [lt=27] MySQL LOGIN(direct_client_ip="192.168.0.127", client_ip=192.168.0.127, tenant_name=sys, tenant_id=1, user_name=proxyro, host_name=%, sessid=3225555614, proxy_sessid=13882346397019341243, sess_create_time=0, from_proxy=true, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=150972045, proxy_capability=1964871, use_ssl=false, c/s protocol="OB_2_0_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:03.835445] INFO [SERVER] process (obmp_init_db.cpp:53) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E1C-0-0] [lt=39] init db(db_name=oceanbase) [2024-08-16 10:20:03.836417] INFO [STORAGE.DUP_TABLE] refresh_dup_tablet_schema_ (ob_dup_table_util.cpp:100) [2525][T1002_DupTbLeas][T1002][Y0-0000000000000000-0-0] [lt=33] find a duplicate ls(ret=0, dup_ls_status_info={tenant_id:1002, ls_id:{id:1079}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:1, is_duplicate:true, is_block_tablet_in:false}}) [2024-08-16 10:20:03.836669] WDIAG [SQL] handle_text_query (ob_sql.cpp:2663) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E1D-0-0] [lt=24][errcode=-5727] fail to handle after get plan(ret=-5727) [2024-08-16 10:20:03.836733] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:549) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E1D-0-0] [lt=20] dump txn free route audit_record(value=5, session->get_sessid()=3225555614, session->get_proxy_sessid()=13882346397019341243) [2024-08-16 10:20:03.837155] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=33] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=7) [2024-08-16 10:20:03.837467] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=56][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa70d9fed70, session_id:3225555614, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:false, reader:{fd:129}, err:5, last_decode_time:1723774803836230, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:868, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:03.837519] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=44] kill and revert session(conn.sessid_=3225555614, proxy_sessid=13882346397019341243, server_id=16, ret=0) [2024-08-16 10:20:03.837532] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=11] sock ref clean, do destroy(*s={this:0x7fa70d9fed70, session_id:3225555614, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:false, reader:{fd:129}, err:5, last_decode_time:1723774803836230, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:868, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:03.837561] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=15] unlock session of tenant(conn.sessid_=3225555614, proxy_sessid=13882346397019341243, conn.tenant_id_=1) [2024-08-16 10:20:03.837574] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31556][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=11] connection close(sessid=3225555614, proxy_sessid=13882346397019341243, tenant_id=1, server_id=16, from_proxy=true, from_java_client=false, c/s protocol="OB_2_0_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCC65F2E1D-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:4, cur_pkt_pos_:6, pkt_rec[0]:{send:obp_mysql_header_{len_:6382, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{send:obp20_header_:{payload_len:14, pkt_seq:0, req_id:4078743, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:578, seq_:2}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[2]:{send:obp20_header_:{payload_len:43, pkt_seq:0, req_id:4078744, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:440, seq_:2}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{receive:obp20_header_:{payload_len:261, pkt_seq:0, req_id:4078745, flag_.flags:62, ext_flags:0}, obp_mysql_header_{len_:257, rec_:261, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[4]:{send:obp_compess_header_{com_len:222, com_seq:1}, obp20_header_:{payload_len:194, pkt_seq:1, req_id:4078745, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:43, seq_:2}, pkt_name:"PKT_ERR", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_compess_header_{com_len:222, com_seq:1}, obp20_header_:{payload_len:194, pkt_seq:1, req_id:4078745, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:151, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}], disconnect_state=4) [2024-08-16 10:20:03.837619] WDIAG [COMMON] get (ob_link_hashmap.h:313) [31841][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=21][errcode=0] uref leak check(node->uref_=1074538688, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:03.837659] INFO [SERVER] run (obmp_disconnect.cpp:82) [31841][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=11] free session successfully(sessid=3225555614) [2024-08-16 10:20:03.837666] INFO [SERVER] run (obmp_disconnect.cpp:89) [31841][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=6] mark session id unused(sessid=3225555614) [2024-08-16 10:20:03.838385] INFO [STORAGE.DUP_TABLE] execute_for_dup_ls_ (ob_dup_table_util.cpp:164) [2525][T1002_DupTbLeas][T1002][Y0-0000000000000000-0-0] [lt=56] ls not leader(cur_ls_ptr->get_ls_id()={id:1079}) [2024-08-16 10:20:03.846326] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:03.846462] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:03.846488] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:03.846499] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:03.847322] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:03.848251] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:03.853954] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.853983] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=26] handle task succ(id={id:1260}) [2024-08-16 10:20:03.853998] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.854014] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.854044] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.860733] INFO [SERVER] try_reload_schema (ob_server_schema_updater.cpp:440) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=14] schedule fetch new schema task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:03.860757] INFO [SERVER] do_heartbeat_event (ob_heartbeat.cpp:268) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=23] try reload schema success(schema_version=1722933080671200, refresh_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_ret=0) [2024-08-16 10:20:03.860763] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:255) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E7F-0-0] [lt=10] [REFRESH_SCHEMA] start to process schema refresh task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:03.860784] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:280) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E7F-0-0] [lt=17] [REFRESH_SCHEMA] local schema info is newer, no need to refresh schema(ret=0, ret="OB_SUCCESS", local_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:03.863467] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:03.863726] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=28] Wash time detail, (compute_wash_size_time=113, refresh_score_time=68, wash_time=15) [2024-08-16 10:20:03.867257] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=24] Cache wash clean map node details(ret=0, clean_node_count=95, clean_time=0, clean_start_pos=2705302, clean_num=62914) [2024-08-16 10:20:03.877576] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [31754][T1_LockWaitMgr][T1][Y0-0000000000000000-0-0] [lt=23] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-08-16 10:20:03.884895] INFO [COMMON] process (ob_rpc_processor_simple.cpp:2405) [13775][T1002_L0_G19][T1002][YB42C0A8007D-00061CCC999B2CED-0-0] [lt=33] handle ObQueryLSIsValidMember requeset(arg_={tenant_id_:1002, self_addr_:"192.168.0.125:2882", ls_array_:[{id:1260}]}) [2024-08-16 10:20:03.884929] INFO [CLOG] get_member_gc_stat (ob_log_handler.cpp:1509) [13775][T1002_L0_G19][T1002][YB42C0A8007D-00061CCC999B2CED-0-0] [lt=28] get_member_gc_stat(is_valid_member=true, stat=1, member_list=3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, learner_list={learner_num:0, learner_array:[]}, addr="192.168.0.125:2882", this={role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}) [2024-08-16 10:20:03.893771] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2520][T1002_TenantMet][T1002][Y0-0000000000000000-0-0] [lt=50] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:17374, free_obj_hold_cnt:593, allocator used:71436792, allocator total:73453184}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:1994, free_obj_hold_cnt:6, allocator used:131182000, allocator total:131182000}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():63488, full allocator used:1600, full allocator total:129024}, tablets_mem=204764208, tablets_mem_limit=1288490180, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:7, allocator used:17024, allocator total:65408}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:106, free_obj_hold_cnt:5057, allocator used:10243392, allocator total:10596096}, pending_cnt=0, wait_gc_count=0, tablet count=19368) [2024-08-16 10:20:03.893833] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF135-0-0] [lt=37][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509888, log_task={header:{begin_lsn:{lsn:18544226334}, end_lsn:{lsn:18544226456}, log_id:53509888, min_scn:{val:1723774803384133004, v:0}, max_scn:{val:1723774803384133004, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544226212}, prev_proposal_id:7, committed_end_lsn:{lsn:18544226334}, data_checksum:1913811101, accum_checksum:3598899534, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774803793582, freeze_ts:1723774803793582, submit_ts:1723774803793629, flushed_ts:1723774803794262, gen_to_freeze cost time:0, gen_to_submit cost time:47, submit_to_flush cost time:633}, fs_cb_begin_ts=1723774803893819, log_life_time=100237) [2024-08-16 10:20:03.894434] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=34] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=851, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:03.902843] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.902865] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:03.902876] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.902886] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.902900] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.911343] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2417][T1002_Occam][T1002][YB42C0A80080-00061CCCBF7D3862-0-0] [lt=22] for each ls(succ_num=4, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:03.926583] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=23] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:03.934781] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80081-00061EFF86EC941B-0-0] [lt=32][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:03.946416] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:03.946578] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=34] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:03.946595] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:03.946603] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:03.948325] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:03.954045] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:03.954102] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=54] handle task succ(id={id:1260}) [2024-08-16 10:20:03.954114] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:03.954128] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:03.954146] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:03.961857] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=13] ====== check clog disk timer task ====== [2024-08-16 10:20:03.961887] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=22] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:03.961917] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=11] cannot_recycle_log_size statistics(cannot_recycle_log_size=391656086, threshold=6088116142, need_update_checkpoint_scn=false) [2024-08-16 10:20:03.965298] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2417][T1002_Occam][T1002][YB42C0A80080-00061CCCBF7D3864-0-0] [lt=34] for each ls(succ_num=4, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:03.980276] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE646-0-0] [lt=43] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=14, avg_log_submit_to_flush_time=755, avg_log_submit_to_slide_time=1217) [2024-08-16 10:20:03.980309] INFO [PALF] read (log_cache.cpp:100) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=29] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:03.984143] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [31728][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=9] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:20:03.993854] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF13B-0-0] [lt=27][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509889, log_task={header:{begin_lsn:{lsn:18544226456}, end_lsn:{lsn:18544226578}, log_id:53509889, min_scn:{val:1723774803884217000, v:0}, max_scn:{val:1723774803884217000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544226334}, prev_proposal_id:7, committed_end_lsn:{lsn:18544226456}, data_checksum:1913811101, accum_checksum:3596867082, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774803893791, freeze_ts:1723774803893792, submit_ts:1723774803893799, flushed_ts:1723774803894508, gen_to_freeze cost time:1, gen_to_submit cost time:8, submit_to_flush cost time:709}, fs_cb_begin_ts=1723774803993840, log_life_time=100049) [2024-08-16 10:20:03.996168] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=10] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:03.996205] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=31] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:03.996329] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:03.996351] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=15][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:03.996362] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=7][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:03.996370] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:04.002915] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.002951] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=33] handle task succ(id={id:1260}) [2024-08-16 10:20:04.002962] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.002972] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.002986] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.012512] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=47] gc stale ls task succ [2024-08-16 10:20:04.012765] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=4] gc stale ls task succ [2024-08-16 10:20:04.012802] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=25] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:04.012852] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=40] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:04.012862] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=8] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:04.016121] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:04.016645] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:04.037264] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=25] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=6) [2024-08-16 10:20:04.046483] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:04.046644] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:04.046677] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1220) [2024-08-16 10:20:04.046692] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:04.046700] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:04.046716] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:04.048431] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=25][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:04.054112] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.054134] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:04.054146] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.054155] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.054169] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.055262] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=50] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:04.055804] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849AFF-0-0] [lt=47] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=10, avg_log_submit_to_flush_time=936, avg_log_submit_to_slide_time=19057) [2024-08-16 10:20:04.060959] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [2663][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=29] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,1,False,{False,},False,False,True}]) [2024-08-16 10:20:04.062401] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [31642][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=25] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-08-16 10:20:04.065223] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:04.065252] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:04.067733] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=33] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:04.068099] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=35] Wash time detail, (compute_wash_size_time=138, refresh_score_time=92, wash_time=25) [2024-08-16 10:20:04.072716] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Cache wash clean map node details(ret=0, clean_node_count=89, clean_time=0, clean_start_pos=2768216, clean_num=62914) [2024-08-16 10:20:04.079328] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=30] PNIO [ratelimit] time: 1723774804079326, bytes: 1496437102940, bw: 0.135969 MB/s, add_ts: 1001305, add_bytes: 142760 [2024-08-16 10:20:04.079397] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8636-0-0] [lt=22] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:04.080846] INFO [LIB] stat (utility.h:1197) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=100] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=46, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:04.080869] INFO [LIB] stat (utility.h:1197) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=22] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=6, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:04.087386] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1723774804087384, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007619, add_bytes: 0 [2024-08-16 10:20:04.103006] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.103030] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:04.103042] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.103054] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.103087] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.128561] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2659][T1001_Occam][T1001][YB42C0A80080-00061CCCBF8D3862-0-0] [lt=37] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:04.128607] WDIAG [OCCAM] dump_statistics (ob_vtable_event_recycle_buffer.h:144) [2659][T1001_Occam][T1001][YB42C0A80080-00061CCCBF8D3863-0-0] [lt=36][errcode=-4006] not init(ret=-4006) [2024-08-16 10:20:04.128626] INFO [MDS] dump_map_holding_item (mds_tenant_service.cpp:556) [2659][T1001_Occam][T1001][YB42C0A80080-00061CCCBF8D3863-0-0] [lt=12] finish scan map holding items(scan_cnt=0) [2024-08-16 10:20:04.128634] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2659][T1001_Occam][T1001][YB42C0A80080-00061CCCBF8D3864-0-0] [lt=6] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:04.139766] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DE-0-0] [lt=98][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:04.140528] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DE-0-0] [lt=25] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:04.146525] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:04.146629] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:04.146643] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:04.146650] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:04.147262] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:04.147354] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=85] get_max_applied_scn(scn={val:18446744073709551615, v:3}, this={ls_id_:{id:1079}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:18446744073709551615, v:3}}) [2024-08-16 10:20:04.148399] INFO [STORAGE.TABLELOCK] garbage_collect_for_all_ls_ (ob_table_lock_service.cpp:213) [2538][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=20] start to check and clear obj lock(ls->get_ls_id()={id:1079}) [2024-08-16 10:20:04.148511] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:120) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=52] tenant info is invalid(ret=-4076, query_ts=0, standby_scn={val:0, v:0}, this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:20:04.148519] INFO [STORAGE.TABLELOCK] garbage_collect_for_all_ls_ (ob_table_lock_service.cpp:213) [2538][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=23] start to check and clear obj lock(ls->get_ls_id()={id:1260}) [2024-08-16 10:20:04.148542] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=29][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:04.148626] INFO [STORAGE.TABLELOCK] garbage_collect_for_all_ls_ (ob_table_lock_service.cpp:213) [2538][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=25] start to check and clear obj lock(ls->get_ls_id()={id:1261}) [2024-08-16 10:20:04.148694] INFO [STORAGE.TABLELOCK] garbage_collect_for_all_ls_ (ob_table_lock_service.cpp:213) [2538][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=11] start to check and clear obj lock(ls->get_ls_id()={id:1262}) [2024-08-16 10:20:04.154183] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.154205] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:04.154213] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.154222] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.154237] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.155830] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B01-0-0] [lt=30] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:04.155854] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B01-0-0] [lt=22] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:04.156383] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=30] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:04.156932] INFO [PALF] read (log_cache.cpp:100) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:04.160521] INFO [CLOG] is_in_sync (ob_log_handler.cpp:546) [31817][T1_L5_G0][T1002][YB42C0A8007F-000616D421E91A0E-0-0] [lt=21] is_in_sync(ret=0, id=1079, is_log_sync=true, is_need_rebuild=false) [2024-08-16 10:20:04.160620] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007F-000616D421E91A0E-0-0] [lt=22][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:04.162702] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=18] ====== tenant freeze timer task ====== [2024-08-16 10:20:04.183792] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=33] gc stale ls task succ [2024-08-16 10:20:04.189539] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF0-0-0] [lt=12][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=509, pos=320, bitmap=NULL) [2024-08-16 10:20:04.194119] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF145-0-0] [lt=10][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509891, log_task={header:{begin_lsn:{lsn:18544226700}, end_lsn:{lsn:18544227103}, log_id:53509891, min_scn:{val:1723774804033739000, v:0}, max_scn:{val:1723774804033739000, v:0}, data_len:347, proposal_id:7, prev_lsn:{lsn:18544226578}, prev_proposal_id:7, committed_end_lsn:{lsn:18544226700}, data_checksum:2436475199, accum_checksum:999357638, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774804079761, freeze_ts:1723774804079762, submit_ts:1723774804079766, flushed_ts:1723774804080653, gen_to_freeze cost time:1, gen_to_submit cost time:5, submit_to_flush cost time:887}, fs_cb_begin_ts=1723774804194105, log_life_time=114344) [2024-08-16 10:20:04.194199] INFO [STORAGE.DUP_TABLE] follower_try_acquire_lease (ob_dup_table_lease.cpp:500) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=59] [DupLeaseLife] The follower can get new lease from this lease log(ret=0, ls_id={id:1079}, lease_log_scn={val:1723774804033739000, v:0}, self_addr="192.168.0.128:2882", acquire_new_lease=false, follower_lease_info_={durable_lease_:{request_ts_:1723774803747028, lease_interval_us_:6000000}, lease_expired_ts_:1723774809747028, last_lease_scn_:{val:1723774804033739000, v:0}, lease_acquire_scn_:{val:1722500589629865001, v:0}}) [2024-08-16 10:20:04.194789] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=49, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:20:04.203091] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.203120] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:20:04.203133] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.203149] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.203170] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.222692] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AB1-0-0] [lt=19] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:04.222732] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AB1-0-0] [lt=26] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774803761389003, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774803761389003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:04.235800] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=28] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:04.236631] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:04.236744] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:04.236954] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:04.236972] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=12] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:04.237259] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=34] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:04.237266] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=9] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:04.237368] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=31] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=5) [2024-08-16 10:20:04.246596] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:04.246694] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:04.246709] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:04.246716] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:04.248584] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:04.254271] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.254292] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:04.254302] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.254312] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.254325] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.270092] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=25] current monitor number(seq_=-1) [2024-08-16 10:20:04.273197] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:04.273466] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash time detail, (compute_wash_size_time=159, refresh_score_time=81, wash_time=16) [2024-08-16 10:20:04.276106] INFO [SQL.PC] inner_cache_evict (ob_ps_cache.cpp:788) [17389][T1002_PsCacheEv][T1002][Y0-0000000000000000-0-0] [lt=40] ps cache evict(tenant_id=1002, stmt_id_map_.size()=8, stmt_info_map_.size()=8, op.get_used_size()=35631, get_mem_high()=144955080, expired_stmt_ids.count()=0, closed_stmt_ids.count()=8) [2024-08-16 10:20:04.276612] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=16] Cache wash clean map node details(ret=0, clean_node_count=107, clean_time=0, clean_start_pos=2831130, clean_num=62914) [2024-08-16 10:20:04.279376] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD863A-0-0] [lt=46] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:04.280000] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=643, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:04.280035] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:04.280058] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=33, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:04.280135] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=45, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:04.280501] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE64F-0-0] [lt=25] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:04.280523] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE64F-0-0] [lt=21] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1216, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:04.282407] INFO [SERVER] fill_tenant_ids_ (ob_all_virtual_tx_stat.cpp:139) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF1-0-0] [lt=28] succeed to get tenant ids(all_tenants_=[1, 1001, 1002]) [2024-08-16 10:20:04.300306] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=15] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:20:04.300349] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=27] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=5202, mem_used=39515136) [2024-08-16 10:20:04.303199] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.303224] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:20:04.303239] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.303251] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.303276] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.307619] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=32] ====== tenant freeze timer task ====== [2024-08-16 10:20:04.307650] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=18][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:04.328971] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE83-0-0] [lt=17] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:04.329019] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE83-0-0] [lt=29] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774803866881004, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774803866881004, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:04.340794] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=100] get all snapshots(ret=0, snapshots.count()=192, snapshots=[{snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213191, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213192, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213193, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213194, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213195, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213196, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213197, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213198, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213199, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213200, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213201, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213202, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213203, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213204, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213205, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213206, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213207, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213208, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213209, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213210, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213211, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213212, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213213, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213214, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213215, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213216, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213217, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213218, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213219, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213220, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213221, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213222, comment:null}]) [2024-08-16 10:20:04.342855] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [2749][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=25] run rewrite rule refresh task(rule_mgr_->tenant_id_=1001) [2024-08-16 10:20:04.346691] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:04.346835] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:04.346855] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:04.346863] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:04.347459] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=677, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:04.347503] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=30] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=32, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:04.347529] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=17] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:04.348100] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7361E-0-0] [lt=11] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:04.348116] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7361E-0-0] [lt=15] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1290, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:04.348662] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=18][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:04.350040] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=22] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:04.350087] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=27] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"sys", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:04.351935] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=23] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:04.352007] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=27] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:04.352030] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=18] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1}) [2024-08-16 10:20:04.352040] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:04.352086] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=21] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:04.352099] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=9] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:04.352156] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=6] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1001}) [2024-08-16 10:20:04.352167] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=10] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"META$1002", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:04.352186] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=8] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"BIGINT":1002}) [2024-08-16 10:20:04.352193] INFO [SERVER] inner_get_next_row (ob_agent_virtual_table.cpp:270) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF2-0-0] [lt=7] mysql compat mode agent do not do convert(ret=0, ret="OB_SUCCESS", input={"VARCHAR":"epean", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}) [2024-08-16 10:20:04.354342] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.354363] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:04.354372] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.354381] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.354400] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.356563] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA6-0-0] [lt=31] push gts task success(*task={:""}) [2024-08-16 10:20:04.374120] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=9] ====== traversal_flush timer task ====== [2024-08-16 10:20:04.374218] INFO [MDS] flush (mds_table_mgr.cpp:195) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=19] [FREEZE]no need do flush cause min_rec_scn is larger than freezing scn(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1079}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:32, ordered_tablets:{{tablet_id:{id:208204}, rec_scn:{val:1723723445809886004, v:0}}, {tablet_id:{id:1152921504606884836}, rec_scn:{val:1723723445809886006, v:0}}, {tablet_id:{id:1152921504606884831}, rec_scn:{val:1723723445809886007, v:0}}, {tablet_id:{id:208210}, rec_scn:{val:1723723445809886008, v:0}}, {tablet_id:{id:1152921504606884610}, rec_scn:{val:1723723445809886009, v:0}}, {tablet_id:{id:1152921504606895583}, rec_scn:{val:1723723445809886011, v:0}}, {tablet_id:{id:1152921504606884840}, rec_scn:{val:1723723445809886012, v:0}}, {tablet_id:{id:1152921504606884615}, rec_scn:{val:1723723445809886013, v:0}}, {tablet_id:{id:1152921504606903842}, rec_scn:{val:1723723446311099001, v:0}}, {tablet_id:{id:208206}, rec_scn:{val:1723723446311099002, v:0}}, {tablet_id:{id:208211}, rec_scn:{val:1723723446311099003, v:0}}, {tablet_id:{id:1152921504606884595}, rec_scn:{val:1723723446311099005, v:0}}, {tablet_id:{id:219872}, rec_scn:{val:1723723446311099006, v:0}}, {tablet_id:{id:208218}, rec_scn:{val:1723723446311099007, v:0}}, {tablet_id:{id:1152921504606884825}, rec_scn:{val:1723723446311099008, v:0}}, {tablet_id:{id:208208}, rec_scn:{val:1723723446311099010, v:0}}, {tablet_id:{id:1152921504606884606}, rec_scn:{val:1723723446311099011, v:0}}, {tablet_id:{id:1152921504606884588}, rec_scn:{val:1723723446311099012, v:0}}, {tablet_id:{id:1152921504606884580}, rec_scn:{val:1723723446311099013, v:0}}, {tablet_id:{id:1152921504606895579}, rec_scn:{val:1723723446311099015, v:0}}, {tablet_id:{id:1152921504606884601}, rec_scn:{val:1723723446311099016, v:0}}, {tablet_id:{id:1152921504606884609}, rec_scn:{val:1723723446311099017, v:0}}, {tablet_id:{id:1152921504606884618}, rec_scn:{val:1723723446311099018, v:0}}, {tablet_id:{id:1152921504606884839}, rec_scn:{val:1723723446311099019, v:0}}, {tablet_id:{id:1152921504606884599}, rec_scn:{val:1723723446311099021, v:0}}, {tablet_id:{id:1152921504606884616}, rec_scn:{val:1723723446311099022, v:0}}, {tablet_id:{id:1152921504606895582}, rec_scn:{val:1723723446812238001, v:0}}, {tablet_id:{id:1152921504606884581}, rec_scn:{val:1723723446812238002, v:0}}, {tablet_id:{id:1152921504606884590}, rec_scn:{val:1723723446812238003, v:0}}, {tablet_id:{id:1152921504606884827}, rec_scn:{val:1723723446812238005, v:0}}, {tablet_id:{id:1152921504606914006}, rec_scn:{val:1723723446812238006, v:0}}, {tablet_id:{id:214096}, rec_scn:{val:1723723446812238007, v:0}}, }, max_consequent_callbacked_scn={val:1723774804033739001, v:0}, *this={this:0x7fa6fa246510, is_inited:true, freezing_scn:{val:1721997282211836001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774801657052003, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:04.375860] INFO [MDS] flush (mds_table_mgr.cpp:195) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=115] [FREEZE]no need do flush cause min_rec_scn is larger than freezing scn(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1260}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:32, ordered_tablets:{{tablet_id:{id:1152921504606868225}, rec_scn:{val:1723723302639709002, v:0}}, {tablet_id:{id:1152921504606912559}, rec_scn:{val:1723723302639709003, v:0}}, {tablet_id:{id:1152921504606871965}, rec_scn:{val:1723723302639709005, v:0}}, {tablet_id:{id:212886}, rec_scn:{val:1723723302639709006, v:0}}, {tablet_id:{id:1152921504606858436}, rec_scn:{val:1723723302639709007, v:0}}, {tablet_id:{id:212255}, rec_scn:{val:1723723302639709009, v:0}}, {tablet_id:{id:1152921504606876671}, rec_scn:{val:1723723302639709010, v:0}}, {tablet_id:{id:1152921504606910457}, rec_scn:{val:1723723302639709012, v:0}}, {tablet_id:{id:214006}, rec_scn:{val:1723723302639709013, v:0}}, {tablet_id:{id:1152921504606912491}, rec_scn:{val:1723723302639709014, v:0}}, {tablet_id:{id:1152921504606890057}, rec_scn:{val:1723723303141598001, v:0}}, {tablet_id:{id:1152921504606848015}, rec_scn:{val:1723723303141598002, v:0}}, {tablet_id:{id:1152921504606847616}, rec_scn:{val:1723723303141598004, v:0}}, {tablet_id:{id:1152921504606913427}, rec_scn:{val:1723723303141598005, v:0}}, {tablet_id:{id:1152921504606915480}, rec_scn:{val:1723723303141598006, v:0}}, {tablet_id:{id:1152921504606889072}, rec_scn:{val:1723723303141598008, v:0}}, {tablet_id:{id:1152921504606915220}, rec_scn:{val:1723723303141598009, v:0}}, {tablet_id:{id:1152921504606866874}, rec_scn:{val:1723723303141598010, v:0}}, {tablet_id:{id:200529}, rec_scn:{val:1723723303141598012, v:0}}, {tablet_id:{id:1152921504606870508}, rec_scn:{val:1723723303141598013, v:0}}, {tablet_id:{id:1152921504606903696}, rec_scn:{val:1723723303141598015, v:0}}, {tablet_id:{id:1152921504606896782}, rec_scn:{val:1723723303647138001, v:0}}, {tablet_id:{id:1152921504606898434}, rec_scn:{val:1723723303647138002, v:0}}, {tablet_id:{id:1152921504606899144}, rec_scn:{val:1723723303647138004, v:0}}, {tablet_id:{id:1152921504606908791}, rec_scn:{val:1723723303829693001, v:0}}, {tablet_id:{id:1152921504606871378}, rec_scn:{val:1723723303829693002, v:0}}, {tablet_id:{id:1152921504606870897}, rec_scn:{val:1723723303829693003, v:0}}, {tablet_id:{id:1152921504606877002}, rec_scn:{val:1723723303829693004, v:0}}, {tablet_id:{id:1152921504606886744}, rec_scn:{val:1723723303829693006, v:0}}, {tablet_id:{id:1152921504606875352}, rec_scn:{val:1723723303829693007, v:0}}, {tablet_id:{id:204192}, rec_scn:{val:1723723303829693009, v:0}}, {tablet_id:{id:1152921504606897887}, rec_scn:{val:1723723304142080001, v:0}}, }, max_consequent_callbacked_scn={val:1723774803925095004, v:0}, *this={this:0x7fa6bd53e510, is_inited:true, freezing_scn:{val:1723690100831412000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1260}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202608224000, v:0}, clog_base_lsn:{lsn:8723619840}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774801657052003, v:0}, tablet_change_checkpoint_scn:{val:1723719123293042000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723639313794159001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1260, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:04.376405] INFO [MDS] flush (mds_table_mgr.cpp:172) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=95] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1261}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x7fa6fe204510, is_inited:true, freezing_scn:{val:1723752041922970000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1261}, ls_create_status:1, clog_checkpoint_scn:{val:1723752188579788000, v:0}, clog_base_lsn:{lsn:32545812480}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774801657052003, v:0}, tablet_change_checkpoint_scn:{val:1723719128263216001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723617733349446001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:4, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1261}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1261, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:04.377641] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA6-0-0] [lt=21] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:04.377804] INFO [MDS] flush (mds_table_mgr.cpp:195) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=46] [FREEZE]no need do flush cause min_rec_scn is larger than freezing scn(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1262}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:32, ordered_tablets:{{tablet_id:{id:209134}, rec_scn:{val:1723723207973113002, v:0}}, {tablet_id:{id:1152921504606869956}, rec_scn:{val:1723723207973113003, v:0}}, {tablet_id:{id:1152921504606871991}, rec_scn:{val:1723723207973113005, v:0}}, {tablet_id:{id:1152921504606868203}, rec_scn:{val:1723723207973113006, v:0}}, {tablet_id:{id:1152921504606857189}, rec_scn:{val:1723723207973113007, v:0}}, {tablet_id:{id:1152921504606903903}, rec_scn:{val:1723723207973113009, v:0}}, {tablet_id:{id:205824}, rec_scn:{val:1723723207973113010, v:0}}, {tablet_id:{id:207632}, rec_scn:{val:1723723207973113011, v:0}}, {tablet_id:{id:1152921504606888997}, rec_scn:{val:1723723207973113013, v:0}}, {tablet_id:{id:214372}, rec_scn:{val:1723723207973113014, v:0}}, {tablet_id:{id:1152921504606915317}, rec_scn:{val:1723723207973113015, v:0}}, {tablet_id:{id:1152921504606875134}, rec_scn:{val:1723723207973113016, v:0}}, {tablet_id:{id:1152921504606876196}, rec_scn:{val:1723723208473765001, v:0}}, {tablet_id:{id:221202}, rec_scn:{val:1723723208473765002, v:0}}, {tablet_id:{id:1152921504606888619}, rec_scn:{val:1723723208473765003, v:0}}, {tablet_id:{id:1152921504606898978}, rec_scn:{val:1723723208473765005, v:0}}, {tablet_id:{id:1152921504606897298}, rec_scn:{val:1723723208473765006, v:0}}, {tablet_id:{id:1152921504606918529}, rec_scn:{val:1723723208473765007, v:0}}, {tablet_id:{id:1152921504606872365}, rec_scn:{val:1723723208473765009, v:0}}, {tablet_id:{id:1152921504606898136}, rec_scn:{val:1723723208473765010, v:0}}, {tablet_id:{id:210672}, rec_scn:{val:1723723208473765011, v:0}}, {tablet_id:{id:209037}, rec_scn:{val:1723723208473765013, v:0}}, {tablet_id:{id:1152921504606858115}, rec_scn:{val:1723723208473765014, v:0}}, {tablet_id:{id:1152921504606896677}, rec_scn:{val:1723723208473765015, v:0}}, {tablet_id:{id:1152921504606896744}, rec_scn:{val:1723723208473765016, v:0}}, {tablet_id:{id:218140}, rec_scn:{val:1723723208473765018, v:0}}, {tablet_id:{id:1152921504606856887}, rec_scn:{val:1723723208473765019, v:0}}, {tablet_id:{id:1152921504606856837}, rec_scn:{val:1723723208473765020, v:0}}, {tablet_id:{id:1152921504606867226}, rec_scn:{val:1723723208973569001, v:0}}, {tablet_id:{id:1152921504606912513}, rec_scn:{val:1723723208973569002, v:0}}, {tablet_id:{id:1152921504606856874}, rec_scn:{val:1723723208973569003, v:0}}, {tablet_id:{id:1152921504606896355}, rec_scn:{val:1723723208973569004, v:0}}, }, max_consequent_callbacked_scn={val:1723774804261746000, v:0}, *this={this:0x7fa6be4fc510, is_inited:true, freezing_scn:{val:1723656976361279000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774801657052003, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:04.377898] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:336) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=90] succeed to traversal_flush(ret=0, ls_cnt=4) [2024-08-16 10:20:04.384498] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=37] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:04.391192] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1961) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(dag_cnt=0, map_size=0) [2024-08-16 10:20:04.391223] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1971) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=29] dump_dag_status(running_dag_net_map_size=0, blocking_dag_net_list_size=0) [2024-08-16 10:20:04.391235] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(priority="PRIO_COMPACTION_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:04.391246] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(priority="PRIO_HA_HIGH", low_limit=8, up_limit=8, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:04.391252] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(priority="PRIO_COMPACTION_MID", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:04.391259] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(priority="PRIO_HA_MID", low_limit=5, up_limit=5, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:04.391265] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(priority="PRIO_COMPACTION_LOW", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:04.391272] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(priority="PRIO_HA_LOW", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:04.391278] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(priority="PRIO_DDL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:04.391285] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(priority="PRIO_DDL_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:04.391291] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(priority="PRIO_TTL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-08-16 10:20:04.391298] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:04.391308] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:04.391315] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:04.391326] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:04.391348] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=21] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:04.391355] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:04.391360] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:04.391364] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:04.391367] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:04.391371] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:04.391377] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-08-16 10:20:04.391385] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-08-16 10:20:04.391391] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:04.391397] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:04.391404] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:04.391410] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:04.391421] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:04.391442] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=20] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:04.391454] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=11] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, dag_count=0) [2024-08-16 10:20:04.391462] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-08-16 10:20:04.391468] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391484] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=15] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391491] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391506] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=14] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391515] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391522] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391528] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391534] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391540] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391551] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391567] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=15] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391572] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391576] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391580] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391584] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391589] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391600] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391605] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391609] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391613] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391616] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391620] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391624] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391629] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391639] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391644] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391648] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391652] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391668] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=15] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-08-16 10:20:04.391677] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391686] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-08-16 10:20:04.391704] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=17] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-08-16 10:20:04.391711] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-08-16 10:20:04.391716] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-08-16 10:20:04.391720] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:04.391726] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:04.391737] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:04.391742] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:04.391746] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:04.391751] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:04.391762] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=11] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:04.391789] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=27] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:04.391816] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=26] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:04.391835] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=18] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:04.391843] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:04.391860] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=15] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:04.391869] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:04.391875] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:04.391881] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:04.391887] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:04.391891] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-08-16 10:20:04.391898] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-08-16 10:20:04.391911] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=12] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:04.391918] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391942] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=22] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:04.391955] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=13] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391967] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:04.391976] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391982] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:04.391988] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:04.391995] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:04.392001] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:04.392007] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:04.392011] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:04.392015] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:04.392023] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:04.392045] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=20] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:04.392054] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:04.392063] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:04.392099] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=35] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:04.392106] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-08-16 10:20:04.392111] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-08-16 10:20:04.392117] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, dag_count=0) [2024-08-16 10:20:04.392133] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=15] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, scheduled_task_count=0) [2024-08-16 10:20:04.392142] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, dag_count=0) [2024-08-16 10:20:04.392149] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, scheduled_task_count=0) [2024-08-16 10:20:04.392165] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=15] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, dag_count=0) [2024-08-16 10:20:04.392172] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-08-16 10:20:04.392176] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, dag_count=0) [2024-08-16 10:20:04.392180] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-08-16 10:20:04.392186] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, dag_count=0) [2024-08-16 10:20:04.392198] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=11] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, scheduled_task_count=0) [2024-08-16 10:20:04.392203] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status[DAG_NET](type="DAG_NET_MIGRATION", dag_net_count=0) [2024-08-16 10:20:04.392207] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_PREPARE_MIGRATION", dag_net_count=0) [2024-08-16 10:20:04.392210] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_COMPLETE_MIGRATION", dag_net_count=0) [2024-08-16 10:20:04.392213] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER", dag_net_count=0) [2024-08-16 10:20:04.392216] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_BACKUP", dag_net_count=0) [2024-08-16 10:20:04.392219] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_RESTORE", dag_net_count=0) [2024-08-16 10:20:04.392222] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TYPE_BACKUP_CLEAN", dag_net_count=0) [2024-08-16 10:20:04.392225] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER_BACKFILL_TX", dag_net_count=0) [2024-08-16 10:20:04.392230] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1991) [2511][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(total_worker_cnt=43, total_running_task_cnt=0, work_thread_num=43, scheduled_task_cnt=0) [2024-08-16 10:20:04.394105] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF151-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1623, avg_group_log_size=147) [2024-08-16 10:20:04.394153] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF151-0-0] [lt=32][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509893, log_task={header:{begin_lsn:{lsn:18544227225}, end_lsn:{lsn:18544227347}, log_id:53509893, min_scn:{val:1723774804033739002, v:0}, max_scn:{val:1723774804033739002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544227103}, prev_proposal_id:7, committed_end_lsn:{lsn:18544227225}, data_checksum:1913811101, accum_checksum:1408565250, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774804293959, freeze_ts:1723774804293959, submit_ts:1723774804293966, flushed_ts:1723774804294618, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:652}, fs_cb_begin_ts=1723774804394143, log_life_time=100184) [2024-08-16 10:20:04.403268] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.403288] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:04.403298] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.403307] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.403321] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.425666] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=31] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:04.425774] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=21] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:04.427145] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=27] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1, need_refresh=false, gts_local_cache={srr:[mts=1723774803926545], gts:1723774803926579000, latest_srr:[mts=1723774804427111]}) [2024-08-16 10:20:04.427187] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=26] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_refresh=false, gts_local_cache={srr:[mts=1723774804366653], gts:1723774804364803000, latest_srr:[mts=1723774804427173]}) [2024-08-16 10:20:04.427371] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=12] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_refresh=false, gts_local_cache={srr:[mts=1723774803926780], gts:1723774803925095000, latest_srr:[mts=1723774804427354]}) [2024-08-16 10:20:04.437477] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=20] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=4) [2024-08-16 10:20:04.446751] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] tx gc loop thread is running(MTL_ID()=1002) [2024-08-16 10:20:04.447633] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774801657052003, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:20:04.447709] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=73] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:04.448639] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1260}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202608224000, v:0}, clog_base_lsn:{lsn:8723619840}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774801657052003, v:0}, tablet_change_checkpoint_scn:{val:1723719123293042000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723639313794159001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1260, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:20:04.448737] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:04.448771] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=45] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:04.449503] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=37423891, this={dio_aligned_buf:{buf_write_offset:2948, buf_padding_size:1148, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:45196170, truncate_used_ts:11123521}, log_block_size:67108864, dir_fd:196, io_fd:111}, aligned_buf_len=4096, aligned_buf="", aligned_block_offset=37421056, buf_len=113, total_write_size_=13072, total_write_size_after_dio_=421888, ob_pwrite_used_ts=65593, count=100, each_pwrite_cost=655) [2024-08-16 10:20:04.449717] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1261}, ls_create_status:1, clog_checkpoint_scn:{val:1723752188579788000, v:0}, clog_base_lsn:{lsn:32545812480}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774801657052003, v:0}, tablet_change_checkpoint_scn:{val:1723719128263216001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723617733349446001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:4, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1261}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1261, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:20:04.449782] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=63] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:04.450795] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774801657052003, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:20:04.450841] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=44] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:04.454431] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.454453] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:04.454464] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.454475] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.454492] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.477039] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:04.477367] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=28] Wash time detail, (compute_wash_size_time=137, refresh_score_time=78, wash_time=16) [2024-08-16 10:20:04.479492] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD863E-0-0] [lt=29] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1342, avg_group_log_size=122) [2024-08-16 10:20:04.481121] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=23] Cache wash clean map node details(ret=0, clean_node_count=90, clean_time=0, clean_start_pos=2894044, clean_num=62914) [2024-08-16 10:20:04.494097] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF157-0-0] [lt=26] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1623) [2024-08-16 10:20:04.494134] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF157-0-0] [lt=23] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=11, avg_log_submit_to_flush_time=737, avg_log_submit_to_slide_time=100063) [2024-08-16 10:20:04.494148] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF157-0-0] [lt=12][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509894, log_task={header:{begin_lsn:{lsn:18544227347}, end_lsn:{lsn:18544227469}, log_id:53509894, min_scn:{val:1723774804385127000, v:0}, max_scn:{val:1723774804385127000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544227225}, prev_proposal_id:7, committed_end_lsn:{lsn:18544227347}, data_checksum:1913811101, accum_checksum:3786030217, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774804394061, freeze_ts:1723774804394061, submit_ts:1723774804394066, flushed_ts:1723774804394701, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:635}, fs_cb_begin_ts=1723774804494124, log_life_time=100063) [2024-08-16 10:20:04.494176] INFO [PALF] read (log_cache.cpp:100) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1623, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:04.503347] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.503371] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:04.503383] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.503392] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.503406] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.526297] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=55] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=6974, mem_used=12478464) [2024-08-16 10:20:04.533679] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=24] start do ls ha handler(ls_id_array_=[{id:1079}, {id:1261}, {id:1260}, {id:1262}]) [2024-08-16 10:20:04.546851] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:04.546970] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=33] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:04.546992] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:04.547000] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:04.548810] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:04.554501] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.554522] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:04.554533] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.554544] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.554559] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.556138] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B09-0-0] [lt=31] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:04.569410] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=13] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 294.00) [2024-08-16 10:20:04.569436] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=20] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:04.569508] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=33] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569534] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=24] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569541] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569546] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569552] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569565] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=12] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569572] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569577] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569581] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569585] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569590] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569594] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569605] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569610] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569616] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.569626] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:04.571195] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E0953-0-0] [lt=19] current clock is (clock_=0, max_seq_=0, mem_limit=1643328294, cur_mem_hold=289406976, attr_.tenant_id_=1002) [2024-08-16 10:20:04.575338] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [31738][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=51] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:940, allocator used:3737440, allocator total:3859072}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:140, allocator used:9182740, allocator total:9182740}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=13041812, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:63, allocator used:124992, allocator total:130816}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:04.591519] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=19][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:04.593827] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=28] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:04.593934] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=20][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:04.593955] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=20][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:04.593963] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=7] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:04.595004] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=34, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:04.603418] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.603440] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:04.603451] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.603461] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.603479] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.607469] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=29] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:04.610368] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [31774][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=9] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:20:04.610391] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:230) [31774][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=16] [tabletchange] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12638) [2024-08-16 10:20:04.617872] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=29] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60693, mem_used=137263104) [2024-08-16 10:20:04.637540] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=21] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=3) [2024-08-16 10:20:04.646925] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:04.646958] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:04.646965] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:04.646969] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:04.648896] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=30][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:04.653473] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=33] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:04.654576] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.654597] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:04.654608] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.654619] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.654633] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.656862] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=543, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:04.656996] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=14] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=39, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:04.657442] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7441A-0-0] [lt=39] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:04.657463] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7441A-0-0] [lt=20] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10857, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:04.681631] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:04.681904] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=117, refresh_score_time=66, wash_time=16) [2024-08-16 10:20:04.685323] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Cache wash clean map node details(ret=0, clean_node_count=99, clean_time=0, clean_start_pos=2956958, clean_num=62914) [2024-08-16 10:20:04.703491] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.703518] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:20:04.703528] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.703537] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.703551] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.746996] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:04.747116] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=51, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:04.747165] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:04.747186] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:04.747194] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:04.747202] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:04.748452] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE65D-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=6, avg_log_gen_to_submit_time=71, avg_log_submit_to_flush_time=797, avg_log_submit_to_slide_time=1246) [2024-08-16 10:20:04.748866] INFO [ARCHIVE] print_self (ob_ls_task.cpp:402) [2568][T1002_ArcSrv][T1002][YB42C0A80080-00061CCC9251D0C5-0-0] [lt=61] print ls archive task(id={id:1260}, tenant_id=1002, station={key:{incarnation:1, dest_id:1003, round:4}, lease:{lease_id:29, lease_start_ts:0, lease_end_ts:0}}, round_start_scn={val:1720659999678271000, v:0}, dest={has_encount_error:false, is_worm:false, max_no_limit_lsn:{lsn:8469425422}, max_archived_info:{offset:{lsn:8828025141}, scn:{val:1723774722817369003, v:0}, piece:{interval_us:86400000000, genesis_scn:{val:1720659999678271000, v:0}, piece_id:44, base_piece_id:8}}, max_seq_log_offset:{lsn:8857829376}, max_fetch_info:{offset:{lsn:8828025141}, scn:{val:1723774722817369003, v:0}, piece:{interval_us:86400000000, genesis_scn:{val:1720659999678271000, v:0}, piece_id:44, base_piece_id:8}}, piece_min_lsn:{lsn:8821470072}, archive_file_id:132, archive_file_offset:6555069, piece_dir_exist:true, wait_send_task_count:0, tasks:}) [2024-08-16 10:20:04.748969] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=27][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:04.754646] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.754671] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:04.754681] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.754691] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.754705] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.762536] INFO [STORAGE.TRANS] rollback_tx (ob_tx_api.cpp:339) [2568][T1002_ArcSrv][T1001][YB42C0A80080-00061CCC9251D0C5-0-0] [lt=50] rollback tx(ret=0, *this={is_inited_:true, tenant_id_:1001, this:0x7fa6e0004030}, tx={this:0x7fa704c12fe0, tx_id:{txid:283711680}, state:7, addr:"192.168.0.128:2882", tenant_id:1001, session_id:3225579948, assoc_session_id:3225579948, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:1723774804754473, op_sn:3, alloc_ts:1723774804755543, active_ts:1723774804755543, commit_ts:-1, finish_ts:1723774804762044, timeout_us:29999986, lock_timeout_us:-1, expire_ts:1723774834755529, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:1713182723, cluster_version:17180000768, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:-6002, commit_expire_ts:0, commit_task_.is_registered():false, ref:2}) [2024-08-16 10:20:04.763126] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=4] LogLoopThread round_cost_time(us)(round_cost_time=20) [2024-08-16 10:20:04.794420] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF169-0-0] [lt=37] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:04.794476] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF169-0-0] [lt=31] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=9, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:04.794484] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF169-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=100124, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:04.794493] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF169-0-0] [lt=7][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509897, log_task={header:{begin_lsn:{lsn:18544227713}, end_lsn:{lsn:18544227835}, log_id:53509897, min_scn:{val:1723774804385127003, v:0}, max_scn:{val:1723774804385127003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544227591}, prev_proposal_id:7, committed_end_lsn:{lsn:18544227713}, data_checksum:1913811101, accum_checksum:2021666911, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774804694271, freeze_ts:1723774804694271, submit_ts:1723774804694276, flushed_ts:1723774804694886, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:610}, fs_cb_begin_ts=1723774804794466, log_life_time=100195) [2024-08-16 10:20:04.795063] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=58] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1501) [2024-08-16 10:20:04.803558] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.803579] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:04.803590] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.803600] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.803614] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.837653] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=25] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=2) [2024-08-16 10:20:04.847091] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:04.847211] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:04.847228] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:04.847235] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:04.847948] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1625) [2024-08-16 10:20:04.849043] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:04.854712] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.854732] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:04.854741] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.854751] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.854765] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.872122] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [2782][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=8] ====== checkpoint timer task ====== [2024-08-16 10:20:04.872146] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [2782][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=18] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:20:04.885759] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:04.886088] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=52] Wash time detail, (compute_wash_size_time=151, refresh_score_time=85, wash_time=16) [2024-08-16 10:20:04.889431] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=16] Cache wash clean map node details(ret=0, clean_node_count=100, clean_time=0, clean_start_pos=3019872, clean_num=62914) [2024-08-16 10:20:04.895021] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=49] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=585, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:04.903617] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.903650] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=30] handle task succ(id={id:1260}) [2024-08-16 10:20:04.903664] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.903676] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.903699] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.927728] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=14] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:04.947141] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:04.947238] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:04.947262] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:04.947289] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:04.949130] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=46][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:04.954788] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:04.954808] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:04.954817] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:04.954827] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:04.954840] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:04.963292] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF_DUMP](palf_id=1262, self="192.168.0.128:2882", [SlidingWindow]={palf_id:1262, self:"192.168.0.128:2882", lsn_allocator:{max_log_id:34459394, max_lsn:8394849804, max_scn:{val:1723774804761912001, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:8394849804}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:8352906764}}, last_submit_lsn:{lsn:8394849682}, last_submit_end_lsn:{lsn:8394849804}, last_submit_log_id:34459394, last_submit_log_pid:2, max_flushed_lsn:{lsn:8394849682}, max_flushed_end_lsn:{lsn:8394849804}, max_flushed_log_pid:2, committed_end_lsn:{lsn:8394849804}, last_slide_log_id:34459394, last_slide_scn:{val:1723774804761912001, v:0}, last_slide_lsn:{lsn:8394849682}, last_slide_end_lsn:{lsn:8394849804}, last_slide_log_pid:2, last_slide_log_accum_checksum:40726317, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x7fa6be52c530}, [StateMgr]={this:0x7fa6be5302b0, self:"192.168.0.128:2882", palf_id:1262, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:2}, leader:"192.168.0.133:2882", leader_epoch:6, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:34459395, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x7fa6be536b70, allow_vote:true}, [ConfigMgr]={palf_id:1262, self:"192.168.0.128:2882", alive_paxos_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744209401, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182753433, flag:0}, alive_paxos_replica_num:3, log_ms_meta:{version:3, proposal_id:2, prev:{version:1, config:{config_version:{proposal_id:2, config_seq:398}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744209401, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182753433, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:29241, lock_time:1723646524328842}}, curr:{version:1, config:{config_version:{proposal_id:2, config_seq:399}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744209401, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182753433, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29241, lock_time:1723646524499878}}, prev_log_proposal_id:2, prev_lsn:{lsn:7750282582}, prev_mode_pid:2}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:2, config_seq:399}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1723774802261226, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1723774802261226, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x7fa6be52dc70}, [ModeMgr]={palf_id:1262, self:"192.168.0.128:2882", applied_mode_meta:{version:1, proposal_id:2, mode_version:0, access_mode:1, ref_scn:{val:1720516541224759000, v:0}}, accepted_mode_meta:{version:1, proposal_id:2, mode_version:0, access_mode:1, ref_scn:{val:1720516541224759000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:2, mode_version:0, access_mode:1, ref_scn:{val:1720516541224759000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1262, is_inited:true, min_block_max_scn:{val:1721957131723543002, v:0}, min_block_id:59, min_block_min_scn:{val:1721929793432294003, v:0}, base_lsn_for_block_gc:{lsn:8253886464}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:2}, log_config_meta:{version:3, proposal_id:2, prev:{version:1, config:{config_version:{proposal_id:2, config_seq:398}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744209401, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182753433, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:29241, lock_time:1723646524328842}}, curr:{version:1, config:{config_version:{proposal_id:2, config_seq:399}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744209401, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182753433, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29241, lock_time:1723646524499878}}, prev_log_proposal_id:2, prev_lsn:{lsn:7750282582}, prev_mode_pid:2}, log_snapshot_meta:{version:1, base_lsn:{lsn:8253886464}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:2, mode_version:0, access_mode:1, ref_scn:{val:1720516541224759000, v:0}}}, log_meta_storage:{log_tail:{lsn:1048576}, readable_log_tail:{lsn:1048576}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:8455200768}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1262}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1262/meta", dir_fd:202, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:66056192, block_header_serialize_buf_:0x7fa6be532438, flashback_version:0}, log_storage:{log_tail:{lsn:8394849804}, readable_log_tail:{lsn:8394849804}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:8388096000}, min_scn:{val:1723770206317513001, v:0}, curr_block_id:125, palf_id:1262}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1262/log", dir_fd:203, min_block_id:59, max_block_id:126, curr_writable_block_id:125}, logical_block_size_:67104768, curr_block_writable_size_:60350964, block_header_serialize_buf_:0x7fa6be5340b8, flashback_version:0}, palf_epoch:4, last_purge_throttling_ts:1723516779808590, this:0x7fa6be5309f0}, [Reconfirm]={palf_id:1262, self:"192.168.0.128:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x7fa6be530430}) [2024-08-16 10:20:04.963506] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=207][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1262, self="192.168.0.128:2882", tmp_server={server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}) [2024-08-16 10:20:04.963525] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=17] [PALF_DUMP](palf_id=1262, self="192.168.0.128:2882", ack_info_list=[]) [2024-08-16 10:20:04.963558] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=28][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1262, self="192.168.0.128:2882", tmp_server={server:"192.168.0.125:2882", timestamp:1720516876539671, flag:0}) [2024-08-16 10:20:04.963574] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=14] current log_task status(palf_id=1262, self="192.168.0.128:2882", log_id=34459395, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[]) [2024-08-16 10:20:04.971171] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007C-000616D3CE4B970B-0-0] [lt=16][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:04.994525] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF175-0-0] [lt=50][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509899, log_task={header:{begin_lsn:{lsn:18544227957}, end_lsn:{lsn:18544228079}, log_id:53509899, min_scn:{val:1723774804886314000, v:0}, max_scn:{val:1723774804886314000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544227835}, prev_proposal_id:7, committed_end_lsn:{lsn:18544227957}, data_checksum:1913811101, accum_checksum:2489925134, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774804894473, freeze_ts:1723774804894473, submit_ts:1723774804894477, flushed_ts:1723774804895091, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:614}, fs_cb_begin_ts=1723774804994512, log_life_time=100039) [2024-08-16 10:20:04.996495] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=6] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:04.996537] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=30] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:04.996682] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:04.996705] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=17][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:04.996714] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:04.996728] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=12][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:05.003647] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.003674] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:20:05.003707] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=30] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.003720] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.003736] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.012582] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=25] gc stale ls task succ [2024-08-16 10:20:05.012683] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=5] gc stale ls task succ [2024-08-16 10:20:05.012714] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=20] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:05.012728] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=9] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:05.012741] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=11] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:05.016409] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:05.017100] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:05.029004] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C1F-0-0] [lt=51] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774805028946) [2024-08-16 10:20:05.037772] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=32] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=1) [2024-08-16 10:20:05.047232] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:05.047341] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1625, avg_group_log_size=162) [2024-08-16 10:20:05.047366] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1625) [2024-08-16 10:20:05.047385] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:05.047404] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:05.047421] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:05.049203] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:05.054863] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.054883] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:05.054893] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.054902] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.054927] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.055932] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=32] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:05.057608] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74426-0-0] [lt=42] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=10, avg_log_submit_to_flush_time=614, avg_log_submit_to_slide_time=1050) [2024-08-16 10:20:05.060009] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80080-00061CCCC6B436C2-0-0] [lt=30][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:05.062292] INFO [STORAGE.TRANS] run1 (ob_black_list.cpp:194) [31842][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=19] ls blacklist refresh finish(cost_time=17069) [2024-08-16 10:20:05.065318] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=58][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:05.065360] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=41][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:05.079901] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=27] PNIO [ratelimit] time: 1723774805079892, bytes: 1496437279238, bw: 0.168036 MB/s, add_ts: 1000566, add_bytes: 176298 [2024-08-16 10:20:05.079938] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD864A-0-0] [lt=28] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:05.081162] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE667-0-0] [lt=17] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=11, avg_log_submit_to_flush_time=882, avg_log_submit_to_slide_time=9862) [2024-08-16 10:20:05.081215] INFO [PALF] read (log_cache.cpp:100) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=37] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1333, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:05.081300] INFO [LIB] stat (utility.h:1197) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=41] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=29, stat_interval=1000000, avg (count/cost)=41, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:05.081332] INFO [LIB] stat (utility.h:1197) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=30] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=29, stat_interval=1000000, avg (count/cost)=8, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:05.089877] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:05.090174] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=39] Wash time detail, (compute_wash_size_time=147, refresh_score_time=70, wash_time=16) [2024-08-16 10:20:05.093462] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Cache wash clean map node details(ret=0, clean_node_count=89, clean_time=0, clean_start_pos=3082786, clean_num=62914) [2024-08-16 10:20:05.094999] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=26] PNIO [ratelimit] time: 1723774805094997, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007613, add_bytes: 0 [2024-08-16 10:20:05.095565] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF17B-0-0] [lt=28][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509900, log_task={header:{begin_lsn:{lsn:18544228079}, end_lsn:{lsn:18544228201}, log_id:53509900, min_scn:{val:1723774804886314001, v:0}, max_scn:{val:1723774804886314001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544227957}, prev_proposal_id:7, committed_end_lsn:{lsn:18544228079}, data_checksum:1913811101, accum_checksum:3879058177, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774804994490, freeze_ts:1723774804994490, submit_ts:1723774804994495, flushed_ts:1723774804995106, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:611}, fs_cb_begin_ts=1723774805095554, log_life_time=101064) [2024-08-16 10:20:05.103723] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.103745] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:05.103756] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.103766] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.103783] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.105753] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [2783][T1001_Flush][T1001][Y0-0000000000000000-0-0] [lt=8] ====== traversal_flush timer task ====== [2024-08-16 10:20:05.105777] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:338) [2783][T1001_Flush][T1001][Y0-0000000000000000-0-0] [lt=18] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:20:05.142917] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DF-0-0] [lt=67][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:05.143457] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7DF-0-0] [lt=26] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:05.147305] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:05.147464] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:05.147483] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:05.147488] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:05.148098] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=31] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1625) [2024-08-16 10:20:05.149272] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:05.154959] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.155008] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=46] handle task succ(id={id:1260}) [2024-08-16 10:20:05.155031] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.155101] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=68] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.155128] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.155882] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=37] current clock is (clock_=0, max_seq_=0, mem_limit=1643328294, cur_mem_hold=289406976, attr_.tenant_id_=1002) [2024-08-16 10:20:05.155926] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=27] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1723774779233879, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1723774805155847, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-08-16 10:20:05.156551] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B15-0-0] [lt=66] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1211, avg_group_log_size=121) [2024-08-16 10:20:05.156591] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B15-0-0] [lt=38] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1211) [2024-08-16 10:20:05.157105] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1211) [2024-08-16 10:20:05.157605] INFO [PALF] read (log_cache.cpp:100) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:05.183872] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=22] gc stale ls task succ [2024-08-16 10:20:05.184547] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [2774][T1001_LCLSender][T1001][Y0-0000000000000000-0-0] [lt=28] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=1135, total_destructed_detector=1135, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x7fa704be42b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-08-16 10:20:05.187037] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [2566][T1002_ArcSender][T1002][YB42C0A80080-00061CCC9231D0C5-0-0] [lt=19][errcode=-4018] try free send task failed(ret=-4018) [2024-08-16 10:20:05.190210] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCC998AE0AA-0-0] [lt=9] receive register_req from acceptor, ignore(self="192.168.0.128:2882", palf_id=1260, child={server:"192.168.0.125:2882", region:"sys_region", register_time_us:1723774805189152, keepalive_ts:-1}) [2024-08-16 10:20:05.190242] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCC998AE0AA-0-0] [lt=29] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1260, self="192.168.0.128:2882", child={server:"192.168.0.125:2882", region:"sys_region", register_time_us:1723774805189152, keepalive_ts:-1}, is_to_leader=true) [2024-08-16 10:20:05.193807] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF7-0-0] [lt=7][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=8, len=98, pos=70, bitmap=NULL) [2024-08-16 10:20:05.199219] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF181-0-0] [lt=12][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509901, log_task={header:{begin_lsn:{lsn:18544228201}, end_lsn:{lsn:18544228323}, log_id:53509901, min_scn:{val:1723774804886314002, v:0}, max_scn:{val:1723774804886314002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544228079}, prev_proposal_id:7, committed_end_lsn:{lsn:18544228201}, data_checksum:1913811101, accum_checksum:2429097809, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774805095532, freeze_ts:1723774805095532, submit_ts:1723774805095537, flushed_ts:1723774805096161, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:624}, fs_cb_begin_ts=1723774805199204, log_life_time=103672) [2024-08-16 10:20:05.199862] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=37, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:20:05.203793] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.203813] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:05.203824] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.203833] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.203847] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.233555] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AB2-0-0] [lt=33] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:05.233613] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AB2-0-0] [lt=34] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774804761912003, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774804761912003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:05.236681] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=15] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:05.236827] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=19] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:05.236929] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=16] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:05.236941] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=11] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:05.237200] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=10] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:05.237308] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=14] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:05.237731] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=24] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:05.237900] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=26] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=0) [2024-08-16 10:20:05.238911] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2520][T1002_TenantMet][T1002][Y0-0000000000000000-0-0] [lt=48] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:17374, free_obj_hold_cnt:593, allocator used:71436792, allocator total:73453184}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:1994, free_obj_hold_cnt:6, allocator used:131182000, allocator total:131182000}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():63488, full allocator used:1600, full allocator total:129024}, tablets_mem=204764208, tablets_mem_limit=1288490180, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:7, allocator used:17024, allocator total:65408}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:106, free_obj_hold_cnt:5057, allocator used:10243392, allocator total:10596096}, pending_cnt=0, wait_gc_count=0, tablet count=19368) [2024-08-16 10:20:05.244945] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD16DAE256-0-0] [lt=54] receive register_req from acceptor, ignore(self="192.168.0.128:2882", palf_id=1260, child={server:"192.168.0.133:2882", region:"sys_region", register_time_us:1723774805242762, keepalive_ts:-1}) [2024-08-16 10:20:05.244972] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD16DAE256-0-0] [lt=25] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1260, self="192.168.0.128:2882", child={server:"192.168.0.133:2882", region:"sys_region", register_time_us:1723774805242762, keepalive_ts:-1}, is_to_leader=true) [2024-08-16 10:20:05.247389] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:05.247508] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:05.247524] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:05.247532] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:05.249349] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:05.255036] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.255057] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:05.255067] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.255099] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=31] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.255114] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.270216] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=22] current monitor number(seq_=-1) [2024-08-16 10:20:05.279986] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD864C-0-0] [lt=12] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1658) [2024-08-16 10:20:05.280650] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=825, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:05.280677] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1658) [2024-08-16 10:20:05.280700] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=30, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:05.280755] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=35, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:05.281137] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7442D-0-0] [lt=20] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:05.281153] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7442D-0-0] [lt=15] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10705, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:05.293922] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:05.294248] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=38] Wash time detail, (compute_wash_size_time=127, refresh_score_time=78, wash_time=23) [2024-08-16 10:20:05.294312] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=0, clean_start_pos=3145700, clean_num=62914) [2024-08-16 10:20:05.300519] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=11] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:20:05.300544] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=22] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=5219, mem_used=39515136) [2024-08-16 10:20:05.303885] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.303910] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:05.303922] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.303933] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.303953] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.340828] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE84-0-0] [lt=35] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:05.340888] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE84-0-0] [lt=42] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774805152682001, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774805152682001, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:05.347461] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:05.347618] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:05.347642] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:05.347651] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:05.348314] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=41] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=703, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:05.348383] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=29, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:05.348433] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=15, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:05.348763] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7363D-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:05.348780] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7363D-0-0] [lt=16] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1312, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:05.349425] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:05.355122] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.355144] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:05.355155] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.355168] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.355196] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.381211] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=18] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574640, offset=42147329}, end_cursor:ObLogCursor{file_id=65, log_id=18574641, offset=42147570}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751714050, len:511, log_data_len:241, seq:18574640, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:05.381959] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=34] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574641, offset=42147570}, end_cursor:ObLogCursor{file_id=65, log_id=18574642, offset=42147819}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751660050, len:4366, log_data_len:249, seq:18574641, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:05.383527] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA7-0-0] [lt=17] push gts task success(*task={:""}) [2024-08-16 10:20:05.384601] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=22] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:05.390141] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA7-0-0] [lt=19] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:05.392519] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=51] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574642, offset=42147819}, end_cursor:ObLogCursor{file_id=65, log_id=18574643, offset=42148068}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751724050, len:4117, log_data_len:249, seq:18574642, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:05.393086] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=29] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574643, offset=42148068}, end_cursor:ObLogCursor{file_id=65, log_id=18574644, offset=42148317}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75175c050, len:3868, log_data_len:249, seq:18574643, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:05.394853] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF18D-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1614, avg_group_log_size=146) [2024-08-16 10:20:05.394914] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF18D-0-0] [lt=45][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509904, log_task={header:{begin_lsn:{lsn:18544228839}, end_lsn:{lsn:18544228961}, log_id:53509904, min_scn:{val:1723774804886314005, v:0}, max_scn:{val:1723774804886314005, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544228436}, prev_proposal_id:7, committed_end_lsn:{lsn:18544228839}, data_checksum:1913811101, accum_checksum:3109783825, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774805294772, freeze_ts:1723774805294772, submit_ts:1723774805294778, flushed_ts:1723774805295381, gen_to_freeze cost time:0, gen_to_submit cost time:6, submit_to_flush cost time:603}, fs_cb_begin_ts=1723774805394904, log_life_time=100132) [2024-08-16 10:20:05.400152] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [2562][T1002_TabletShe][T1002][Y0-0000000000000000-0-0] [lt=9] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:20:05.400209] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:99) [2562][T1002_TabletShe][T1002][Y0-0000000000000000-0-0] [lt=44] [emptytablet] succeed to change tablet to empty shell(ret=0, ret="OB_SUCCESS", ls_cnt=4, times=12267) [2024-08-16 10:20:05.403953] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.403974] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:05.403984] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.403995] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.404010] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.405963] INFO pn_ratelimit (group.c:551) [31557][IngressService][T0][Y0-0000000000000000-0-0] [lt=22] PNIO set ratelimit as 9223372036854775807 bytes/s, grp_id=2 [2024-08-16 10:20:05.425798] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=28] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:05.425889] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=27] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:05.428779] INFO [STORAGE.TRANS] alloc (ob_ts_response_handler.cpp:108) [31439][pnio1][T0][YB42C0A80080-00061CCCCD6206C5-0-0] [lt=23] ts response task statistics(alloc_count=170000602, free_count=85000300) [2024-08-16 10:20:05.439519] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=20] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=2705302, replace_num=31457) [2024-08-16 10:20:05.439556] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=34] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=10) [2024-08-16 10:20:05.447544] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:05.447656] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:05.447679] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:05.447686] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:05.449516] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:05.455206] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.455230] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:05.455242] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.455253] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.455274] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.456246] INFO [STORAGE.TRANS] generate_new_version (ob_tenant_weak_read_server_version_mgr.cpp:120) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=26] [WRS] update tenant weak read server version(tenant_id=1001, server_version={version:{val:1723774800926440000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0, epoch_tstamp:1723774805456230}, version_delta=4529805) [2024-08-16 10:20:05.466264] INFO [STORAGE] runTimerTask (ob_locality_manager.cpp:708) [31516][LocalityReload][T0][Y0-0000000000000000-0-0] [lt=19] runTimer to refresh locality_info(ret=0) [2024-08-16 10:20:05.468527] INFO [SHARE] set_server_locality_array (ob_server_locality_cache.cpp:346) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=28] set_server_locality_array success(server_locality_array=[{inited:true, addr:"192.168.0.124:2882", zone:"zone1", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1713946685756789, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.125:2882", zone:"zone1", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1720514289430943, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.126:2882", zone:"zone1", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1722932391226462, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.127:2882", zone:"zone2", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1713947174711570, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.128:2882", zone:"zone2", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1720514727760575, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.129:2882", zone:"zone2", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1722932391566566, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.131:2882", zone:"zone3", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1714007235497482, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.133:2882", zone:"zone3", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1720516415939965, server_stop_time:0, server_status:1}, {inited:true, addr:"192.168.0.134:2882", zone:"zone3", zone_type:0, idc:"default_idc", region:"sys_region", is_active:true, start_service_time:1722932391301527, server_stop_time:0, server_status:1}], has_readonly_zone=false) [2024-08-16 10:20:05.468617] INFO [SHARE] load_region (ob_locality_table_operator.cpp:346) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=71] get_primary_zone_prioriry(primary_zone=zone1,zone2,zone3, tenant_primary_region_array=[{region:"sys_region", region_priority:0, zone_array:["zone1", "zone2", "zone3"]}], locality_info={version:0, local_region:"sys_region", local_zone:"zone2", local_idc:"default_idc", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone1", "zone2", "zone3"]}], locality_zone_array:[]}) [2024-08-16 10:20:05.468667] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=48] add_locality_zone success(locality_zone={tenant_id:1, region_priority:0}) [2024-08-16 10:20:05.468719] INFO [SHARE] load_region (ob_locality_table_operator.cpp:346) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=41] get_primary_zone_prioriry(primary_zone=zone1,zone2,zone3, tenant_primary_region_array=[{region:"sys_region", region_priority:0, zone_array:["zone1", "zone2", "zone3"]}], locality_info={version:0, local_region:"sys_region", local_zone:"zone2", local_idc:"default_idc", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone1", "zone2", "zone3"]}], locality_zone_array:[{tenant_id:1, region_priority:0}]}) [2024-08-16 10:20:05.468732] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=12] add_locality_zone success(locality_zone={tenant_id:1001, region_priority:0}) [2024-08-16 10:20:05.468742] INFO [SHARE] load_region (ob_locality_table_operator.cpp:346) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=3] get_primary_zone_prioriry(primary_zone=zone1,zone2,zone3, tenant_primary_region_array=[{region:"sys_region", region_priority:0, zone_array:["zone1", "zone2", "zone3"]}], locality_info={version:0, local_region:"sys_region", local_zone:"zone2", local_idc:"default_idc", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone1", "zone2", "zone3"]}], locality_zone_array:[{tenant_id:1, region_priority:0}, {tenant_id:1001, region_priority:0}]}) [2024-08-16 10:20:05.468772] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=29] add_locality_zone success(locality_zone={tenant_id:1002, region_priority:0}) [2024-08-16 10:20:05.468788] INFO [SHARE] load_region (ob_locality_table_operator.cpp:373) [31515][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=11] load region(ret=0, locality_info={version:0, local_region:"sys_region", local_zone:"zone2", local_idc:"default_idc", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone1", "zone2", "zone3"]}], locality_zone_array:[{tenant_id:1, region_priority:0}, {tenant_id:1001, region_priority:0}, {tenant_id:1002, region_priority:0}]}) [2024-08-16 10:20:05.478662] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [2767][T1001_LockWaitM][T1001][Y0-0000000000000000-0-0] [lt=51] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-08-16 10:20:05.483339] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8650-0-0] [lt=49] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1658, avg_group_log_size=165) [2024-08-16 10:20:05.490519] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [21116][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7E0-0-0] [lt=18] [RS_MGR] force set rs list(master_rs="192.168.0.126:2882") [2024-08-16 10:20:05.490553] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [21116][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7E0-0-0] [lt=25] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"192.168.0.128:2882", server_id:16, rs_addr:"192.168.0.126:2882", rs_server_status:1, epoch_id:11}, hb_response={zone:"zone2", server:"192.168.0.128:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1720514727760575, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=36) [2024-08-16 10:20:05.494762] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:05.494991] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF193-0-0] [lt=26] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1614) [2024-08-16 10:20:05.495030] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF193-0-0] [lt=22] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=618, avg_log_submit_to_slide_time=91015) [2024-08-16 10:20:05.495045] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=33] Wash time detail, (compute_wash_size_time=133, refresh_score_time=90, wash_time=16) [2024-08-16 10:20:05.495096] INFO [PALF] read (log_cache.cpp:100) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1614, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:05.496541] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007D-00061CCCA5024D1E-0-0] [lt=124][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:05.498776] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Cache wash clean map node details(ret=0, clean_node_count=103, clean_time=0, clean_start_pos=0, clean_num=62914) [2024-08-16 10:20:05.499028] INFO print_tenant_cache_info (ob_kvcache_inst_map.cpp:520) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] [CACHE] tenant_id= 1 cache memory info: [CACHE] tenant_id= 1 | cache_name= schema_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 121 | hold_size= 0 [CACHE] tenant_id= 1 | cache_name= tablet_table_cache | cache_size= 8321024 | cache_store_size= 4161536 | cache_map_size= 4159488 | kv_cnt= 22952 | hold_size= 0 [CACHE] tenant_id= 1 | cache_name= vtable_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 38 | hold_size= 0 [CACHE] tenant_id= 1 | cache_name= index_block_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 66 | hold_size= 0 [CACHE] tenant_id= 1 | cache_name= user_block_cache | cache_size= 4160512 | cache_store_size= 2080768 | cache_map_size= 2079744 | kv_cnt= 73 | hold_size= 0 [CACHE] tenant_id= 1 | cache_name= tmp_block_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 1 | hold_size= 0 [CACHE] tenant_id= 1 | cache_name= tx_data_kv_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 1 | hold_size= 0 [CACHE] tenant_id= 1 | cache_name= opt_table_stat_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 118 | hold_size= 0 [CACHE] tenant_id= 1 | cache_name= opt_column_stat_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 544 | hold_size= 0 [CACHE] tenant_id= 1 | cache_name= BACKUP_INDEX_CACHE | cache_size= 0 | cache_store_size= 0 | cache_map_size= 0 | kv_cnt= 0 | hold_size= 0 [2024-08-16 10:20:05.499104] INFO print_tenant_cache_info (ob_kvcache_inst_map.cpp:520) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] [CACHE] tenant_id= 1001 cache memory info: [CACHE] tenant_id= 1001 | cache_name= opt_table_stat_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 42 | hold_size= 0 [CACHE] tenant_id= 1001 | cache_name= opt_column_stat_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 323 | hold_size= 0 [2024-08-16 10:20:05.499165] INFO print_tenant_cache_info (ob_kvcache_inst_map.cpp:520) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=15] [CACHE] tenant_id= 1002 cache memory info: [CACHE] tenant_id= 1002 | cache_name= index_block_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 186 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= user_block_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 137 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= user_row_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 325 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= bf_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 3 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= fuse_row_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 699 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= storage_meta_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 3790 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= tmp_page_cache | cache_size= 4160512 | cache_store_size= 2080768 | cache_map_size= 2079744 | kv_cnt= 217 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= tmp_block_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 2 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= tx_data_kv_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 527 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= opt_table_stat_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 3073 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= opt_column_stat_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 922 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= opt_ds_stat_cache | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 1436 | hold_size= 0 [CACHE] tenant_id= 1002 | cache_name= BACKUP_INDEX_CACHE | cache_size= 6241280 | cache_store_size= 4161536 | cache_map_size= 2079744 | kv_cnt= 46 | hold_size= 0 [2024-08-16 10:20:05.499206] INFO print_current_status (ob_kvcache_hazard_version.cpp:431) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=15] [KVCACHE-HAZARD] hazard version status info: current version: 253476007 | min_version=253476007 | total thread store count: 289 | total nodes count: 103 | [KVCACHE-HAZARD] i= 282 | thread_id= 31419 | inited= 1 | waiting_nodes_count= 103 | last_retire_version=253475904 | acquired_version=18446744073709551615 | [2024-08-16 10:20:05.503977] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=26] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1002, server_version={version:{val:1723774804886314005, v:0}, total_part_count:4, valid_inner_part_count:0, valid_user_part_count:4}, server_version_delta=617660, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.124:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29408395, succ_cluster_heartbeat_count=29408385, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774715049399002, v:0}, local_cluster_delta=90454575, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:20:05.504024] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.504037] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] handle task succ(id={id:1260}) [2024-08-16 10:20:05.504046] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.504055] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.504085] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.506843] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80083-000616E44D0F990E-0-0] [lt=23][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:05.526451] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=29] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=1, size_used=6995, mem_used=12478464) [2024-08-16 10:20:05.533864] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=31] start do ls ha handler(ls_id_array_=[{id:1079}, {id:1260}, {id:1262}, {id:1261}]) [2024-08-16 10:20:05.547523] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=37] new mysql sessid created(conn.sessid_=3225600948, support_ssl=false) [2024-08-16 10:20:05.547549] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E1E-0-0] [lt=4] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12185, add_timestamp:1723774805547520}) [2024-08-16 10:20:05.547557] WDIAG [SHARE.LOCATION] add_update_task (ob_vtable_location_service.cpp:388) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF8-0-0] [lt=21][errcode=-4023] fail to add task(ret=-4023, ret="OB_EAGAIN", task={tenant_id:1, table_id:12185, add_timestamp:1723774805547520}) [2024-08-16 10:20:05.547573] WDIAG [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:444) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF8-0-0] [lt=16][errcode=-4023] add location update task failed(ret=-4023, ret="OB_EAGAIN", task={tenant_id:1, table_id:12185, add_timestamp:1723774805547520}) [2024-08-16 10:20:05.547575] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=22] sm conn init succ(conn.sessid_=3225600948, sess.client_addr_="127.0.0.1:51056") [2024-08-16 10:20:05.547577] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676704-0-0] [lt=61] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=12185, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:05.547590] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=9] accept one succ(*s={this:0x7fa71f01e030, session_id:3225600948, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:129}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:05.547580] WDIAG [SHARE.LOCATION] vtable_get (ob_vtable_location_service.cpp:156) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF8-0-0] [lt=6][errcode=0] add location update task failed(temp_ret=-4023, temp_ret="OB_EAGAIN", tenant_id=1, table_id=12185, vtable_locations=[{table_id:12185, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774775548039, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774775548039, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774775548039, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774775548039, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774775548039, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774775548039, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774775548039, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774775548039, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774775548039, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:05.547620] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:05.547729] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=30] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:05.547753] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:05.547762] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:05.548196] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676704-0-0] [lt=40] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12185, locations=[{table_id:12185, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:05.548286] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676704-0-0] [lt=88] success to process renew task(task={tenant_id:1, table_id:12185, add_timestamp:1723774805547520}, locations=[{table_id:12185, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}, {table_id:12185, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805548166, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:05.549584] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=28][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:05.550677] INFO [SERVER] extract_user_tenant (obmp_connect.cpp:79) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=19] username and tenantname(user_name=ocp_monitor, tenant_name=) [2024-08-16 10:20:05.550717] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31555][sql_nio0][T1][Y0-0000000000000000-0-0] [lt=21] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:05.550842] INFO load_privilege_info (obmp_connect.cpp:532) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24E-0-0] [lt=50] no tenant name set, use default tenant name(tenant_name=sys) [2024-08-16 10:20:05.551429] WDIAG [COMMON] get (ob_link_hashmap.h:313) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24E-0-0] [lt=22][errcode=0] uref leak check(node->uref_=1074538689, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:05.551512] WDIAG [SERVER] report_rt_monitor_node (ob_virtual_sql_plan_monitor.cpp:276) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E1E-0-0] [lt=18][errcode=-4008] rt node iter end(ret=-4008) [2024-08-16 10:20:05.551511] INFO [SERVER] process (obmp_connect.cpp:475) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24E-0-0] [lt=33] MySQL LOGIN(direct_client_ip="127.0.0.1", client_ip=127.0.0.1, tenant_name=sys, tenant_id=1, user_name=ocp_monitor, host_name=%, sessid=3225600948, proxy_sessid=0, sess_create_time=0, from_proxy=false, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=762509, proxy_capability=0, use_ssl=false, c/s protocol="OB_MYSQL_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:05.552105] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [31651][T1_PalfGC][T1][Y0-0000000000000000-0-0] [lt=29] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}) [2024-08-16 10:20:05.552142] WDIAG [SERVER] inner_get_next_row (ob_virtual_sql_plan_monitor.cpp:196) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B56169-0-0] [lt=24][errcode=-4008] fail to switch tenant monitor node list(ret=-4008) [2024-08-16 10:20:05.552714] WDIAG [SERVER] inner_get_next_row (ob_virtual_sql_plan_monitor.cpp:196) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B5616A-0-0] [lt=9][errcode=-4008] fail to switch tenant monitor node list(ret=-4008) [2024-08-16 10:20:05.553215] WDIAG [SERVER] inner_get_next_row (ob_virtual_sql_plan_monitor.cpp:196) [21114][T1_L0_G0][T1][YB42C0A80080-00061CCCC1B5616B-0-0] [lt=17][errcode=-4008] fail to switch tenant monitor node list(ret=-4008) [2024-08-16 10:20:05.553730] WDIAG [SERVER] report_rt_monitor_node (ob_virtual_sql_plan_monitor.cpp:276) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E1F-0-0] [lt=16][errcode=-4008] rt node iter end(ret=-4008) [2024-08-16 10:20:05.553749] WDIAG [SERVER] inner_get_next_row (ob_virtual_sql_plan_monitor.cpp:196) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E1F-0-0] [lt=17][errcode=-4008] fail to switch tenant monitor node list(ret=-4008) [2024-08-16 10:20:05.554133] WDIAG [SERVER] inner_get_next_row (ob_virtual_sql_plan_monitor.cpp:196) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E20-0-0] [lt=7][errcode=-4008] fail to switch tenant monitor node list(ret=-4008) [2024-08-16 10:20:05.555253] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.555275] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:05.555283] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.555292] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.555291] WDIAG [SERVER] report_rt_monitor_node (ob_virtual_sql_plan_monitor.cpp:276) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF8-0-0] [lt=59][errcode=-4008] rt node iter end(ret=-4008) [2024-08-16 10:20:05.555307] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.555491] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa71f1f6030, session_id:3225575535, trace_id:Y0-00061CCCC85CFAF9-0-0, sql_handling_stage:3, sql_initiative_shutdown:false, reader:{fd:134}, err:5, last_decode_time:1723774805554593, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:141996, pending_flag:1, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:05.555520] WDIAG [SERVER] disconnect (obmp_packet_sender.cpp:830) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF9-0-0] [lt=10][errcode=0] server close connection(sessid=3225575535, proxy_sessid=0, stack="0x12a0688c 0xa219970 0xa1e504b 0xa527c01 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:05.555533] INFO [SERVER] process (obmp_quit.h:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF9-0-0] [lt=11] quit [2024-08-16 10:20:05.555528] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=27] kill and revert session(conn.sessid_=3225575535, proxy_sessid=0, server_id=16, ret=0) [2024-08-16 10:20:05.555544] WDIAG [RPC.OBMYSQL] async_write_data (ob_sql_sock_session.cpp:163) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF9-0-0] [lt=10][errcode=-4009] sock has error(ret=-4009) [2024-08-16 10:20:05.555553] WDIAG [SERVER] flush_buffer (obmp_packet_sender.cpp:1032) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF9-0-0] [lt=8][errcode=-4009] write response fail(ret=-4009) [2024-08-16 10:20:05.555558] WDIAG [SERVER] response (obmp_base.cpp:87) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF9-0-0] [lt=5][errcode=-4009] failed to flush_buffer(ret=-4009) [2024-08-16 10:20:05.555564] WDIAG [RPC.FRAME] run (ob_sql_processor.cpp:48) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAF9-0-0] [lt=5][errcode=-4009] response rpc result fail(ret=-4009) [2024-08-16 10:20:05.555563] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=13] can close safely, do destroy(*s={this:0x7fa71f1f6030, session_id:3225575535, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:true, reader:{fd:134}, err:5, last_decode_time:1723774805554593, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:142001, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-08-16 10:20:05.555592] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=14] unlock session of tenant(conn.sessid_=3225575535, proxy_sessid=0, conn.tenant_id_=1) [2024-08-16 10:20:05.555600] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=14][errcode=0] uref leak check(node->uref_=1074538689, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:05.555653] INFO [SERVER] run (obmp_disconnect.cpp:82) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=8] free session successfully(sessid=3225575535) [2024-08-16 10:20:05.555668] INFO [SERVER] run (obmp_disconnect.cpp:89) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=14] mark session id unused(sessid=3225575535) [2024-08-16 10:20:05.555606] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=12] connection close(sessid=3225575535, proxy_sessid=0, tenant_id=1, server_id=16, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCC85CFAF8-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:828, pkt_rec[796]:{send:obp_mysql_header_{len_:9, seq_:8}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[797]:{receive:obp_mysql_header_{len_:148, rec_:1508, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[798]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[799]:{send:obp_mysql_header_{pkt_num_:2780, seq_:3}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[800]:{send:obp_mysql_header_{len_:9, seq_:4}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[801]:{send:obp_mysql_header_{pkt_num_:2741, seq_:5}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[802]:{send:obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[803]:{receive:obp_mysql_header_{len_:2742, rec_:6844, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[804]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[805]:{send:obp_mysql_header_{pkt_num_:2874, seq_:86}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[806]:{send:obp_mysql_header_{len_:9, seq_:87}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[807]:{send:obp_mysql_header_{pkt_num_:2743, seq_:88}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[808]:{send:obp_mysql_header_{len_:9, seq_:89}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[809]:{receive:obp_mysql_header_{len_:145, rec_:6985, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[810]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[811]:{send:obp_mysql_header_{pkt_num_:2747, seq_:3}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[812]:{send:obp_mysql_header_{len_:9, seq_:4}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[813]:{send:obp_mysql_header_{pkt_num_:2741, seq_:5}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[814]:{send:obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[815]:{receive:obp_mysql_header_{len_:2740, rec_:15062, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[816]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[817]:{send:obp_mysql_header_{pkt_num_:2858, seq_:86}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[818]:{send:obp_mysql_header_{len_:9, seq_:87}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[819]:{send:obp_mysql_header_{pkt_num_:707, seq_:105}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[820]:{send:obp_mysql_header_{len_:9, seq_:106}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[821]:{receive:obp_mysql_header_{len_:125, rec_:12446, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[822]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[823]:{send:obp_mysql_header_{pkt_num_:79, seq_:3}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[824]:{send:obp_mysql_header_{len_:9, seq_:4}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[825]:{send:obp_mysql_header_{pkt_num_:2779, seq_:5}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[826]:{send:obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[827]:{receive:obp_mysql_header_{len_:1, rec_:12438, seq_:0}, pkt_name:"COM_QUIT", obp_mysql_header_.type_:1, obp_mysql_header_.is_send_:0}], disconnect_state=3) [2024-08-16 10:20:05.555781] WDIAG [SERVER] inner_get_next_row (ob_virtual_sql_plan_monitor.cpp:196) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E21-0-0] [lt=16][errcode=-4008] fail to switch tenant monitor node list(ret=-4008) [2024-08-16 10:20:05.556780] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B1D-0-0] [lt=16] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:05.572606] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 1032.00) [2024-08-16 10:20:05.572629] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=18] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:05.572698] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=34] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 6144.00, iops: 3.99, rt: 489.50) [2024-08-16 10:20:05.572712] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=12] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:05.572730] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=13] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572745] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=14] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572751] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572757] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572770] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=12] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572777] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572781] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572786] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572790] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572797] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572808] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572824] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=14] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572832] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572839] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572846] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.572853] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:05.584787] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=7] ====== check clog disk timer task ====== [2024-08-16 10:20:05.584819] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=28] get_disk_usage(ret=0, capacity(MB):=2150, used(MB):=0) [2024-08-16 10:20:05.584840] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=10] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=676457349, need_update_checkpoint_scn=false) [2024-08-16 10:20:05.585895] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=6] ====== check clog disk timer task ====== [2024-08-16 10:20:05.585921] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=23] get_disk_usage(ret=0, capacity(MB):=0, used(MB):=0) [2024-08-16 10:20:05.585941] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=14] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=0, need_update_checkpoint_scn=false) [2024-08-16 10:20:05.591485] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=10][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:05.594158] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=26] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:05.594244] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=23][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:05.594265] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=20][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:05.594274] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=8] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:05.595474] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=21, stat_interval=1000000, avg (count/cost)=29, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:05.604114] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.604140] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:20:05.604136] INFO [STORAGE.TRANS] generate_weak_read_timestamp_ (ob_ls_wrs_handler.cpp:188) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=45] get wrs ts(ls_id={id:1079}, delta=217071, log_service_ts=1723774805386765, min_tx_service_ts=4611686018427387, timestamp={val:1723774805386765001, v:0}) [2024-08-16 10:20:05.604151] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.604162] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.604159] INFO [STORAGE.TRANS] print_stat_info (ob_keep_alive_ls_handler.cpp:214) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=18] [Keep Alive Stat] LS Keep Alive Info(tenant_id=1002, LS_ID={id:1079}, Not_Master_Cnt=100, Near_To_GTS_Cnt=0, Other_Error_Cnt=0, Submit_Succ_Cnt=0, last_scn="{val:1723774804886314003, v:0}", last_lsn={lsn:18544228379}, last_gts={val:0, v:0}, min_start_scn="{val:0, v:0}", min_start_status=1, sys_ls_end_scn={val:0, v:0}) [2024-08-16 10:20:05.604177] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.607599] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=21] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:05.618028] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=28] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60695, mem_used=137263104) [2024-08-16 10:20:05.626786] WDIAG [SERVER] do_process (obmp_query.cpp:1018) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E0956-0-0] [lt=24][errcode=0] need disconnect(ret=0, need_disconnect=true) [2024-08-16 10:20:05.639644] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=10] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=9) [2024-08-16 10:20:05.645689] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=16] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:20:05.645733] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=33] [tabletchange] task check ls(ls->get_ls_id()={id:1079}, tablet_persist_trigger=0) [2024-08-16 10:20:05.645751] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=15] [tabletchange] task check ls(ls->get_ls_id()={id:1260}, tablet_persist_trigger=0) [2024-08-16 10:20:05.645763] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=11] [tabletchange] task check ls(ls->get_ls_id()={id:1261}, tablet_persist_trigger=0) [2024-08-16 10:20:05.645769] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=5] [tabletchange] task check ls(ls->get_ls_id()={id:1262}, tablet_persist_trigger=0) [2024-08-16 10:20:05.645775] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:228) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=5] [tabletchange] succeed to gc_tablet(ret=0, ret="OB_SUCCESS", ls_cnt=4, times=11134) [2024-08-16 10:20:05.647693] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:05.647814] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:05.647852] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=35] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:05.647863] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:05.649698] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=39][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:05.655328] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.655351] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:05.655362] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.655373] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.655404] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.657487] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=557, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:05.657595] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=36, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:05.658009] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73647-0-0] [lt=23] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=14, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:05.658038] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73647-0-0] [lt=27] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1074, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:05.661414] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=36] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:05.695000] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF19F-0-0] [lt=11][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509907, log_task={header:{begin_lsn:{lsn:18544229205}, end_lsn:{lsn:18544229327}, log_id:53509907, min_scn:{val:1723774805386765002, v:0}, max_scn:{val:1723774805386765002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544229083}, prev_proposal_id:7, committed_end_lsn:{lsn:18544229205}, data_checksum:1913811101, accum_checksum:1572603279, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774805594888, freeze_ts:1723774805594888, submit_ts:1723774805594895, flushed_ts:1723774805595487, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:592}, fs_cb_begin_ts=1723774805694989, log_life_time=100101) [2024-08-16 10:20:05.699718] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=28] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:05.700010] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=31] Wash time detail, (compute_wash_size_time=119, refresh_score_time=83, wash_time=15) [2024-08-16 10:20:05.702425] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [31775][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=14] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:20:05.702457] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:101) [31775][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=24] [emptytablet] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12638) [2024-08-16 10:20:05.704183] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.704203] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:05.704220] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.704263] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=41] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.704278] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.704602] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=24] Cache wash clean map node details(ret=0, clean_node_count=86, clean_time=0, clean_start_pos=62914, clean_num=62914) [2024-08-16 10:20:05.744477] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E22-0-0] [lt=19] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11004, add_timestamp:1723774805744438}) [2024-08-16 10:20:05.744483] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=68] new mysql sessid created(conn.sessid_=3225659595, support_ssl=false) [2024-08-16 10:20:05.744493] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676705-0-0] [lt=55] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11004, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:05.744502] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAFA-0-0] [lt=6] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11004, add_timestamp:1723774805744439}) [2024-08-16 10:20:05.744546] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=19] sm conn init succ(conn.sessid_=3225659595, sess.client_addr_="127.0.0.1:51058") [2024-08-16 10:20:05.744565] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=11] accept one succ(*s={this:0x7fa71f1f4030, session_id:3225659595, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:134}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:05.744640] INFO [SERVER] extract_user_tenant (obmp_connect.cpp:79) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=32] username and tenantname(user_name=ocp_monitor, tenant_name=) [2024-08-16 10:20:05.744684] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31555][sql_nio0][T1][Y0-0000000000000000-0-0] [lt=19] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:05.744803] INFO load_privilege_info (obmp_connect.cpp:532) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D32-0-0] [lt=60] no tenant name set, use default tenant name(tenant_name=sys) [2024-08-16 10:20:05.745022] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676705-0-0] [lt=19] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11004, locations=[{table_id:11004, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:05.745111] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676705-0-0] [lt=87] success to process renew task(task={tenant_id:1, table_id:11004, add_timestamp:1723774805744438}, locations=[{table_id:11004, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805744994, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:05.745189] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676706-0-0] [lt=64] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11004, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:05.745367] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D32-0-0] [lt=21][errcode=0] uref leak check(node->uref_=1074538690, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:05.745448] INFO [SERVER] process (obmp_connect.cpp:475) [21124][T1_MysqlQueueTh][T1][Y0-00061CCCC1C35D32-0-0] [lt=20] MySQL LOGIN(direct_client_ip="127.0.0.1", client_ip=127.0.0.1, tenant_name=sys, tenant_id=1, user_name=ocp_monitor, host_name=%, sessid=3225659595, proxy_sessid=0, sess_create_time=0, from_proxy=false, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=762509, proxy_capability=0, use_ssl=false, c/s protocol="OB_MYSQL_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:05.745596] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676706-0-0] [lt=12] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11004, locations=[{table_id:11004, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:05.745646] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676706-0-0] [lt=48] success to process renew task(task={tenant_id:1, table_id:11004, add_timestamp:1723774805744439}, locations=[{table_id:11004, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}, {table_id:11004, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805745581, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:05.747781] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:05.747863] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=46, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:05.747902] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:05.747919] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:05.747930] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:05.747937] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:05.749125] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE67C-0-0] [lt=46] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=6, avg_log_gen_to_submit_time=67, avg_log_submit_to_flush_time=720, avg_log_submit_to_slide_time=1211) [2024-08-16 10:20:05.749700] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=13][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:05.749737] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [2674][T1001_LogLoop][T1001][Y0-0000000000000000-0-0] [lt=16] LogLoopThread round_cost_time(us)(round_cost_time=2) [2024-08-16 10:20:05.755409] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.755430] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:05.755440] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.755449] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.755462] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.756707] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E23-0-0] [lt=25] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11048, add_timestamp:1723774805756688}) [2024-08-16 10:20:05.756728] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676707-0-0] [lt=63] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11048, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:05.757212] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676707-0-0] [lt=33] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11048, locations=[{table_id:11048, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:05.757274] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676707-0-0] [lt=60] success to process renew task(task={tenant_id:1, table_id:11048, add_timestamp:1723774805756688}, locations=[{table_id:11048, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}, {table_id:11048, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774805757187, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:05.757449] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=16][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x7fa71f01e030, session_id:3225600948, trace_id:YB42C0A80080-00061CCCC85CFAFB-0-0, sql_handling_stage:10, sql_initiative_shutdown:false, reader:{fd:129}, err:5, last_decode_time:1723774805756988, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:2079, pending_flag:1, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:05.757496] WDIAG [SERVER] disconnect (obmp_packet_sender.cpp:830) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAFB-0-0] [lt=14][errcode=0] server close connection(sessid=3225600948, proxy_sessid=0, stack="0x12a0688c 0xa219970 0xa1e504b 0xa527c01 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:05.757513] INFO [SERVER] process (obmp_quit.h:56) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAFB-0-0] [lt=14] quit [2024-08-16 10:20:05.757518] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=60] kill and revert session(conn.sessid_=3225600948, proxy_sessid=0, server_id=16, ret=0) [2024-08-16 10:20:05.757521] WDIAG [RPC.OBMYSQL] async_write_data (ob_sql_sock_session.cpp:163) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAFB-0-0] [lt=6][errcode=-4009] sock has error(ret=-4009) [2024-08-16 10:20:05.757529] WDIAG [SERVER] flush_buffer (obmp_packet_sender.cpp:1032) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAFB-0-0] [lt=8][errcode=-4009] write response fail(ret=-4009) [2024-08-16 10:20:05.757534] WDIAG [SERVER] response (obmp_base.cpp:87) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAFB-0-0] [lt=5][errcode=-4009] failed to flush_buffer(ret=-4009) [2024-08-16 10:20:05.757539] WDIAG [RPC.FRAME] run (ob_sql_processor.cpp:48) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAFB-0-0] [lt=4][errcode=-4009] response rpc result fail(ret=-4009) [2024-08-16 10:20:05.769530] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=16] ====== tenant freeze timer task ====== [2024-08-16 10:20:05.769564] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=20][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:05.795038] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1A5-0-0] [lt=20] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:05.795119] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1A5-0-0] [lt=53] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=10, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:05.795133] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1A5-0-0] [lt=14] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=91000, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:05.795143] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1A5-0-0] [lt=7][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509908, log_task={header:{begin_lsn:{lsn:18544229327}, end_lsn:{lsn:18544229449}, log_id:53509908, min_scn:{val:1723774805386765003, v:0}, max_scn:{val:1723774805386765003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544229205}, prev_proposal_id:7, committed_end_lsn:{lsn:18544229327}, data_checksum:1913811101, accum_checksum:3865115376, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774805694969, freeze_ts:1723774805694969, submit_ts:1723774805694973, flushed_ts:1723774805695611, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:638}, fs_cb_begin_ts=1723774805795109, log_life_time=100140) [2024-08-16 10:20:05.795693] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1614) [2024-08-16 10:20:05.804263] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.804289] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:20:05.804303] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.804315] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.804290] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=23] get_max_applied_scn(scn={val:1723774805427508002, v:0}, this={ls_id_:{id:1260}, role_:1, proposal_id_:29, palf_committed_end_lsn_:{lsn:8828146516}, last_check_scn_:{val:1723774805427508003, v:0}, max_applied_cb_scn_:{val:1723774805427508002, v:0}}) [2024-08-16 10:20:05.804332] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.804354] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=56] get_min_unreplayed_log_info(lsn={lsn:8469424174}, scn={val:1723617929041268003, v:0}, this={ls_id_:{id:1260}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723617929662670, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:2710542, ret_code_:0}}, next_to_submit_lsn_:{lsn:8469424174}, next_to_submit_scn_:{val:1723617929041268003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516550107534003, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69fc05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8469424052}, end_lsn:{lsn:8469424174}, read_buf:{buf_len_:3682304, buf_:0x7fa69fc05000}, block_size:67104768, log_storage_:0x7fa6bd55b470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723617929041268002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723617929041268002, v:0}, data_checksum:3690972251, flag:0}}, init_mode_version:0, accumulate_checksum:2828504552, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723617929041268002, v:0}}}}}) [2024-08-16 10:20:05.804421] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=57] get_max_applied_scn(scn={val:1723617908966009002, v:0}, this={ls_id_:{id:1261}, role_:2, proposal_id_:4, palf_committed_end_lsn_:{lsn:32276646160}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1723617908966009002, v:0}}) [2024-08-16 10:20:05.804434] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=10] get_min_unreplayed_log_info(lsn={lsn:32601569984}, scn={val:1723774805368552005, v:0}, this={ls_id_:{id:1261}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774805784566, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:344950, ret_code_:0}}, next_to_submit_lsn_:{lsn:32601569984}, next_to_submit_scn_:{val:1723774805368552005, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516552080113001, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd805000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:32601569862}, end_lsn:{lsn:32601569984}, read_buf:{buf_len_:3682304, buf_:0x7fa6cd805000}, block_size:67104768, log_storage_:0x7fa6fe299470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774805368552004, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774805368552004, v:0}, data_checksum:1657047228, flag:1}}, init_mode_version:0, accumulate_checksum:1680833842, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774805368552004, v:0}}}}}) [2024-08-16 10:20:05.804495] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=51] get_max_applied_scn(scn={val:0, v:0}, this={ls_id_:{id:1262}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:0, v:0}}) [2024-08-16 10:20:05.804507] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=9] get_min_unreplayed_log_info(lsn={lsn:8394850780}, scn={val:1723774805263513005, v:0}, this={ls_id_:{id:1262}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774805758051, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10679383, ret_code_:0}}, next_to_submit_lsn_:{lsn:8394850780}, next_to_submit_scn_:{val:1723774805263513005, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516551920929002, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69ec05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8394850658}, end_lsn:{lsn:8394850780}, read_buf:{buf_len_:3682304, buf_:0x7fa69ec05000}, block_size:67104768, log_storage_:0x7fa6be533470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774805263513004, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774805263513004, v:0}, data_checksum:2892311396, flag:0}}, init_mode_version:0, accumulate_checksum:1300091919, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774805263513004, v:0}}}}}) [2024-08-16 10:20:05.839762] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=27] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=8) [2024-08-16 10:20:05.847875] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:05.848015] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=32] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:05.848033] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:05.848041] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:05.848742] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:05.849774] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:05.855471] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.855492] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:05.855503] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.855512] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.855530] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.856498] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=25] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1001, server_version={version:{val:1723774800926440000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0}, server_version_delta=4930054, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.124:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29431381, succ_cluster_heartbeat_count=29431369, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774715706314000, v:0}, local_cluster_delta=90150180, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:20:05.861514] INFO [SERVER] try_reload_schema (ob_server_schema_updater.cpp:440) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=18] schedule fetch new schema task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:05.861536] INFO [SERVER] do_heartbeat_event (ob_heartbeat.cpp:268) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=21] try reload schema success(schema_version=1722933080671200, refresh_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_ret=0) [2024-08-16 10:20:05.861562] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:255) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E80-0-0] [lt=9] [REFRESH_SCHEMA] start to process schema refresh task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:05.861630] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:280) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E80-0-0] [lt=62] [REFRESH_SCHEMA] local schema info is newer, no need to refresh schema(ret=0, ret="OB_SUCCESS", local_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:05.873465] INFO [SERVER] reload_tenant_task (ob_tenant_tablet_ttl_mgr.cpp:1297) [20386][T1002_TTLTablet][T1002][YB42C0A80080-00061CCC99C6C54A-0-0] [lt=49] finish reload tenant task(local_tenant_task_={tenant_id:1002, task_id:-1, is_usr_trigger:false, need_check:false, is_dirty:false, ttl_continue:true, rsp_time:-1, state:4, is_reused:true}, tenant_task={gmt_create:0, gmt_modified:0, tenant_id:18446744073709551615, table_id:18446744073709551615, tablet_id:18446744073709551615, task_id:0, task_start_time:0, task_update_time:0, trigger_type:0, status:19, ttl_del_cnt:0, max_version_del_cnt:0, scan_cnt:0, row_key:"", ret_code:"OB_SUCCESS"}, is_leader=true) [2024-08-16 10:20:05.882127] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1044) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=133] success to update min reserved snapshot(reserved_snapshot=1723773000638406000, duration=1800, snapshot_gc_ts_=1723774800638406000) [2024-08-16 10:20:05.883472] INFO [STORAGE] update_min_reserved_snapshot_for_leader (ob_ls_reserved_snapshot_mgr.cpp:188) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=17] submit reserved snapshot log success(ls_id={id:1260}, new_snapshot_version=1723773000638406000) [2024-08-16 10:20:05.883499] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1071) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=22] update reserved snapshot finished(cost_ts=1374, reserved_snapshot=1723773000638406000) [2024-08-16 10:20:05.895670] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=569, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:05.901441] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2752][T1001_TenantMet][T1001][Y0-0000000000000000-0-0] [lt=91] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=0, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:05.904307] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.904327] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:05.904338] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.904347] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.904362] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.905115] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:05.905409] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=145, refresh_score_time=106, wash_time=16) [2024-08-16 10:20:05.908689] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Cache wash clean map node details(ret=0, clean_node_count=79, clean_time=0, clean_start_pos=125828, clean_num=62914) [2024-08-16 10:20:05.928943] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=30] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:05.943878] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=10] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1002) [2024-08-16 10:20:05.943913] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=25] start lib cache evict(tenant_id=1002, mem_hold=279150592, mem_limit=322122545, cache_obj_num=644, cache_node_num=575) [2024-08-16 10:20:05.943924] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=10] end lib cache evict(tenant_id=1002, cache_evict_num=0, mem_hold=279150592, mem_limit=322122545, cache_obj_num=644, cache_node_num=575) [2024-08-16 10:20:05.943932] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=6] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:05.947669] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [31787][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=30] tx gc loop thread is running(MTL_ID()=1) [2024-08-16 10:20:05.947940] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:05.948037] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:05.948051] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:05.948058] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:05.947478] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=5] Dumping All Cache Objs(alloc_obj_list.count()=644, alloc_obj_list=[{obj_id:197058, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179184}, {obj_id:246944, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:166013}, {obj_id:246946, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175596}, {obj_id:199478, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205804}, {obj_id:249042, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:250289, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174834}, {obj_id:250293, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187797}, {obj_id:250294, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158389}, {obj_id:252064, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179845}, {obj_id:252089, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252117, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188403}, {obj_id:252210, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252318, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191338}, {obj_id:252323, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182962}, {obj_id:252333, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252334, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252336, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252340, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252341, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:252345, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221400}, {obj_id:252346, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284685}, {obj_id:252349, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181199}, {obj_id:252359, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:356776}, {obj_id:252372, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180356}, {obj_id:252377, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252386, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252404, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:382127}, {obj_id:252405, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252406, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:252409, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230382}, {obj_id:252417, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277038}, {obj_id:252425, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158366}, {obj_id:252428, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252440, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213028}, {obj_id:252445, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178214}, {obj_id:252447, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:252458, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:941831}, {obj_id:252459, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252464, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157801}, {obj_id:252475, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176302}, {obj_id:252485, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181511}, {obj_id:252488, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203010}, {obj_id:252489, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197827}, {obj_id:252495, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181747}, {obj_id:252502, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:705639}, {obj_id:252503, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:355214}, {obj_id:252511, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294158}, {obj_id:252513, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217004}, {obj_id:252540, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256131}, {obj_id:252541, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294112}, {obj_id:252554, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219188}, {obj_id:252569, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203562}, {obj_id:252573, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174535}, {obj_id:252586, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179957}, {obj_id:252587, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:254547}, {obj_id:252603, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294123}, {obj_id:252660, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196529}, {obj_id:252664, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294182}, {obj_id:252670, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292094}, {obj_id:252671, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294048}, {obj_id:252673, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211150}, {obj_id:252699, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202759}, {obj_id:252702, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252720, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157744}, {obj_id:252721, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:273791}, {obj_id:252730, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189375}, {obj_id:252749, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:399910}, {obj_id:252750, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:314035}, {obj_id:252764, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299083}, {obj_id:252765, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179021}, {obj_id:252769, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:355879}, {obj_id:252771, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188178}, {obj_id:252786, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237378}, {obj_id:252841, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177408}, {obj_id:252849, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252858, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180522}, {obj_id:252859, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180179}, {obj_id:252862, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6708112}, {obj_id:252864, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252865, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232017}, {obj_id:252868, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232266}, {obj_id:252869, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220840}, {obj_id:252939, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252955, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175994}, {obj_id:252962, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252964, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:349147}, {obj_id:252975, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252980, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252994, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252995, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252998, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253001, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253005, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253006, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253007, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253008, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294264}, {obj_id:253009, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292151}, {obj_id:253015, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298926}, {obj_id:253022, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301178}, {obj_id:253025, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298998}, {obj_id:253035, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298892}, {obj_id:253039, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:351399}, {obj_id:253040, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292087}, {obj_id:253047, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298992}, {obj_id:253050, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:288707}, {obj_id:253053, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204401}, {obj_id:253059, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208911}, {obj_id:253060, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211382}, {obj_id:253061, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200421}, {obj_id:253065, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223579}, {obj_id:253071, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290322}, {obj_id:253072, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250919}, {obj_id:253087, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253090, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:323458}, {obj_id:253091, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272373}, {obj_id:253094, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200393}, {obj_id:253095, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187302}, {obj_id:253096, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206572}, {obj_id:253097, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190787}, {obj_id:253098, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177031}, {obj_id:253102, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272582}, {obj_id:253107, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272526}, {obj_id:253113, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213908}, {obj_id:253114, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201513}, {obj_id:253115, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185332}, {obj_id:253122, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240210}, {obj_id:253124, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245608}, {obj_id:253127, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:372694}, {obj_id:253137, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1957477}, {obj_id:253169, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:270222}, {obj_id:253170, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272447}, {obj_id:253174, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:795070}, {obj_id:253234, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:283358}, {obj_id:253282, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253312, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286729}, {obj_id:253341, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253444, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157703}, {obj_id:253450, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186097}, {obj_id:253451, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:287939}, {obj_id:253452, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258575}, {obj_id:253454, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277108}, {obj_id:253457, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218998}, {obj_id:253458, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:285701}, {obj_id:253459, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253468, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181399}, {obj_id:253471, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:874259}, {obj_id:253473, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232858}, {obj_id:253476, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253477, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253478, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207070}, {obj_id:253480, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250296}, {obj_id:253485, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253486, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201045}, {obj_id:253487, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201415}, {obj_id:253488, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202980}, {obj_id:253489, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225769}, {obj_id:253490, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201728}, {obj_id:253491, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197763}, {obj_id:253497, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253500, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253506, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:11947420}, {obj_id:253507, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228298}, {obj_id:253508, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176153}, {obj_id:253509, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223501}, {obj_id:253510, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176542}, {obj_id:253511, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:253512, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218568}, {obj_id:253513, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177328}, {obj_id:253514, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177907}, {obj_id:253515, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184844}, {obj_id:253516, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253517, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177168}, {obj_id:253518, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:253519, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253520, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:253521, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:253522, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:253523, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253524, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253525, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253526, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253527, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:162334}, {obj_id:253529, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253530, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181547}, {obj_id:253531, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181340}, {obj_id:253532, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192983}, {obj_id:253533, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253534, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186898}, {obj_id:253535, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180995}, {obj_id:253536, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253537, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253538, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:253539, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253540, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223911}, {obj_id:253541, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253542, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221984}, {obj_id:253543, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186900}, {obj_id:253544, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213278}, {obj_id:253545, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6705734}, {obj_id:253546, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6739530}, {obj_id:253547, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202652}, {obj_id:253548, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253549, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:236673}, {obj_id:253550, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253551, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253553, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253554, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:367141}, {obj_id:253555, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189381}, {obj_id:253556, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253557, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253558, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205876}, {obj_id:253559, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253560, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189377}, {obj_id:253561, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253562, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237070}, {obj_id:253564, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205271}, {obj_id:253565, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238959}, {obj_id:253566, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292100}, {obj_id:253567, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:287824}, {obj_id:253568, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6741367}, {obj_id:253570, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187799}, {obj_id:253571, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253572, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175560}, {obj_id:253573, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175558}, {obj_id:253574, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:160948}, {obj_id:253576, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253578, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253579, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253580, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235030}, {obj_id:253581, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234962}, {obj_id:253582, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200857}, {obj_id:253583, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253584, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175142}, {obj_id:253585, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174672}, {obj_id:253588, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292043}, {obj_id:253589, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253590, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253591, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253594, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292025}, {obj_id:253595, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253596, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253601, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:383372}, {obj_id:253605, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292063}, {obj_id:253608, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290453}, {obj_id:253610, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253611, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:369179}, {obj_id:253614, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282531}, {obj_id:253615, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292282}, {obj_id:253616, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253617, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197701}, {obj_id:253618, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:476477}, {obj_id:253621, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297717}, {obj_id:253628, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:244896}, {obj_id:253630, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203464}, {obj_id:253631, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196891}, {obj_id:253632, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:253633, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186088}, {obj_id:253634, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181148}, {obj_id:253635, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253638, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:268654}, {obj_id:253640, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201594}, {obj_id:253641, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:265278}, {obj_id:253642, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290670}, {obj_id:253645, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253649, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292134}, {obj_id:253650, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206516}, {obj_id:253653, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290821}, {obj_id:253655, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238325}, {obj_id:253658, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198080}, {obj_id:253659, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216604}, {obj_id:253660, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178528}, {obj_id:253661, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:243178}, {obj_id:253664, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291977}, {obj_id:253665, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216113}, {obj_id:253666, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242569}, {obj_id:253667, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253668, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207760}, {obj_id:253669, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253672, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292096}, {obj_id:253677, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178509}, {obj_id:253681, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292081}, {obj_id:253682, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182898}, {obj_id:253683, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225569}, {obj_id:253691, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208341}, {obj_id:253692, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211962}, {obj_id:253694, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:379579}, {obj_id:253695, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185463}, {obj_id:253696, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253697, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253698, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194454}, {obj_id:253699, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185281}, {obj_id:253700, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277153}, {obj_id:253701, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196647}, {obj_id:253702, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220331}, {obj_id:253703, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253704, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228523}, {obj_id:253705, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184067}, {obj_id:253706, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175893}, {obj_id:253707, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189895}, {obj_id:253708, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251032}, {obj_id:253709, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189423}, {obj_id:253710, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:334817}, {obj_id:253711, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189681}, {obj_id:253712, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209440}, {obj_id:253713, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188174}, {obj_id:253714, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253715, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253716, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174832}, {obj_id:253723, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:932120}, {obj_id:253724, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:262469}, {obj_id:253725, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:332574}, {obj_id:253726, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240754}, {obj_id:253727, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210719}, {obj_id:253728, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253729, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:253730, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185730}, {obj_id:253731, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250192}, {obj_id:253732, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177398}, {obj_id:253733, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191786}, {obj_id:253734, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177488}, {obj_id:253735, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276369}, {obj_id:253736, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219016}, {obj_id:253737, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:510255}, {obj_id:253740, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197323}, {obj_id:253743, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234930}, {obj_id:253747, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210144}, {obj_id:253749, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253750, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253751, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227821}, {obj_id:253753, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:302184}, {obj_id:253754, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253755, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183280}, {obj_id:253756, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253757, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282524}, {obj_id:253758, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200791}, {obj_id:253759, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:384669}, {obj_id:253760, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:382379}, {obj_id:253761, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:873919}, {obj_id:253762, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:870651}, {obj_id:253763, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:516499}, {obj_id:253764, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253765, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253766, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204396}, {obj_id:253767, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222427}, {obj_id:253768, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253769, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221512}, {obj_id:253770, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:554188}, {obj_id:253771, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247245}, {obj_id:253772, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253773, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196589}, {obj_id:253774, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187792}, {obj_id:253775, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198057}, {obj_id:253776, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279996}, {obj_id:253777, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221914}, {obj_id:253779, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180817}, {obj_id:253780, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179884}, {obj_id:253782, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225027}, {obj_id:253784, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250315}, {obj_id:253791, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292016}, {obj_id:253792, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291532}, {obj_id:253793, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180590}, {obj_id:253796, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250098}, {obj_id:253797, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180228}, {obj_id:253798, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176856}, {obj_id:253799, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195067}, {obj_id:253800, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253801, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177751}, {obj_id:253802, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208961}, {obj_id:253805, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240180}, {obj_id:253808, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256449}, {obj_id:253809, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:449159}, {obj_id:253810, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188388}, {obj_id:253811, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199543}, {obj_id:253812, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195231}, {obj_id:253813, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253814, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253815, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214158}, {obj_id:253816, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253817, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:288727}, {obj_id:253820, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291994}, {obj_id:253821, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235062}, {obj_id:253827, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175823}, {obj_id:253828, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174483}, {obj_id:253829, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217784}, {obj_id:253830, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177814}, {obj_id:253831, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:444256}, {obj_id:253832, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195622}, {obj_id:253833, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253834, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:497014}, {obj_id:253835, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253836, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:261672}, {obj_id:253837, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:435156}, {obj_id:253838, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:516719}, {obj_id:253839, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200167}, {obj_id:253840, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271961}, {obj_id:253841, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194596}, {obj_id:253842, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205257}, {obj_id:253843, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174397}, {obj_id:253845, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253846, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253847, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253848, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224798}, {obj_id:253849, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223872}, {obj_id:253850, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224553}, {obj_id:253851, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221103}, {obj_id:253852, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219500}, {obj_id:253853, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219425}, {obj_id:253854, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253855, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240470}, {obj_id:253856, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253857, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253858, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220267}, {obj_id:253859, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220212}, {obj_id:253861, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290476}, {obj_id:253864, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:547504}, {obj_id:253865, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233724}, {obj_id:253866, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218340}, {obj_id:253867, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197559}, {obj_id:253868, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253869, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217547}, {obj_id:253870, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235957}, {obj_id:253871, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237233}, {obj_id:253872, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:280008}, {obj_id:253873, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208674}, {obj_id:253874, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190144}, {obj_id:253875, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240637}, {obj_id:253876, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187604}, {obj_id:253878, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241654}, {obj_id:253879, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253880, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253881, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176921}, {obj_id:253882, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188072}, {obj_id:253883, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:246455}, {obj_id:253884, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176040}, {obj_id:253886, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253887, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253890, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, adde [2024-08-16 10:20:05.948369] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=888] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:05.949858] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:05.950966] INFO [SHARE] get_server_tenant_configs (ob_unit_getter.cpp:231) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=5] get_server_tenant_configs(unit_infos=[{unit:unit_id:1014, resource_pool_id:1006, unit_group_id:1004, zone:"zone2", server:"192.168.0.128:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:8, max_cpu:8, memory_size:"7GB", log_disk_size:"21GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, pool:{resource_pool_id:1006, name:"pool_epean_zone2_qcd", unit_count:2, unit_config_id:1059, zone_list:["zone2"], tenant_id:1002, replica_type:0}}]) [2024-08-16 10:20:05.951025] INFO [SHARE] get_compat_mode (ob_unit_getter.cpp:810) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=55] jx_debug: get tenant compatibility mode(tenant_id=1002, compat_mode=0) [2024-08-16 10:20:05.951038] INFO [SHARE] divide_meta_tenant (ob_unit_resource.cpp:658) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=11] divide meta tenant resource finish(ret=0, ret="OB_SUCCESS", meta_resource={min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"2.1GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}, user_resource={min_cpu:8, max_cpu:8, memory_size:"6GB", log_disk_size:"18.9GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}, unit_resource={min_cpu:8, max_cpu:8, memory_size:"7GB", log_disk_size:"21GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}) [2024-08-16 10:20:05.951057] INFO [SHARE] divide_meta_tenant (ob_unit_getter.cpp:125) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=17] divide meta tenant finish(ret=0, ret="OB_SUCCESS", meta_tenant_config={tenant_id:1001, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"2.1GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:0, is_removed:false}, user_config={tenant_id:1002, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:8, max_cpu:8, memory_size:"6GB", log_disk_size:"18.9GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:0, is_removed:false}) [2024-08-16 10:20:05.951225] INFO [COMMON] update_io_clock (ob_io_mclock.cpp:454) [31598][OmtNodeBalancer][T1002][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=65] update other group clock success(index=0, unit_config={min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, cur_config={deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}) [2024-08-16 10:20:05.951265] INFO [COMMON] update_basic_io_config (ob_io_manager.cpp:918) [31598][OmtNodeBalancer][T1002][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=39] update basic io config success(tenant_id_=1002, io_config_={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:0, max_percent:0, weight_percent:0}]}, io_clock_={is_inited_:true, group_clocks:[], other_clock:{is_inited_:true, is_stopped_:false, reservation_clock:{iops:9223372036854775807, last_ns:1723774805392627000}, is_unlimited:false, limitation_clock:{iops:9223372036854775807, last_ns:1723774805392627000}, proportion_clock:{iops:8, last_ns:8227606537638699712}}, unit_clock:{iops:9223372036854775807, last_ns:1723774805392627000}, io_config_:{group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, io_usage_:{doing_request_count:[0:0]}}) [2024-08-16 10:20:05.951377] INFO [COMMON] update_io_clock (ob_io_mclock.cpp:454) [31598][OmtNodeBalancer][T1001][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=38] update other group clock success(index=0, unit_config={min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, cur_config={deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}) [2024-08-16 10:20:05.951393] INFO [COMMON] update_basic_io_config (ob_io_manager.cpp:918) [31598][OmtNodeBalancer][T1001][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=15] update basic io config success(tenant_id_=1001, io_config_={group_num_:0, memory_limit_:1073741824, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, io_config={group_num_:0, memory_limit_:1073741824, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:0, max_percent:0, weight_percent:0}]}, io_clock_={is_inited_:true, group_clocks:[], other_clock:{is_inited_:true, is_stopped_:false, reservation_clock:{iops:9223372036854775807, last_ns:0}, is_unlimited:false, limitation_clock:{iops:9223372036854775807, last_ns:0}, proportion_clock:{iops:8, last_ns:0}}, unit_clock:{iops:9223372036854775807, last_ns:0}, io_config_:{group_num_:0, memory_limit_:1073741824, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, io_usage_:{doing_request_count:[0:0]}}) [2024-08-16 10:20:05.951428] INFO [SERVER.OMT] refresh_hidden_sys_memory (ob_tenant_node_balancer.cpp:427) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=25] update hidden sys tenant memory succeed (allowed_mem_limit=1073741824) [2024-08-16 10:20:05.955539] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:05.955559] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:05.955569] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:05.955577] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:05.955590] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:05.956428] INFO [SHARE] refresh_resource_plan (ob_resource_plan_manager.cpp:122) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=10] refresh resource plan success(tenant_id=1, plan_name=, directives=[]) [2024-08-16 10:20:05.958912] INFO [SHARE] refresh_resource_user_mapping_rule (ob_resource_mapping_rule_manager.cpp:159) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=44] refresh resource user mapping rule(tenant_id=1, plan=, user_rules=[]) [2024-08-16 10:20:05.961353] INFO [SHARE] refresh_resource_function_mapping_rule (ob_resource_mapping_rule_manager.cpp:196) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=26] refresh_resource_function_mapping_rule(tenant_id=1, plan=, rules=[]) [2024-08-16 10:20:05.961382] INFO [SHARE] refresh_resource_mapping_rule (ob_resource_mapping_rule_manager.cpp:100) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=27] refresh resource mapping rule success(tenant_id=1, plan=) [2024-08-16 10:20:05.962017] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=9] ====== check clog disk timer task ====== [2024-08-16 10:20:05.962049] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=23] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:05.962110] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=46] cannot_recycle_log_size statistics(cannot_recycle_log_size=391667589, threshold=6088116142, need_update_checkpoint_scn=false) [2024-08-16 10:20:05.963552] INFO [SQL.DTL] calc_max_buffer (ob_dtl_fc_server.cpp:159) [31598][OmtNodeBalancer][T1][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=7] trace tenant dfc parameters(max_parallel_cnt_=20, max_blocked_buffer_size_=6553600, max_buffer_size_=13107200) [2024-08-16 10:20:05.963622] INFO [SQL.DTL] auto_free_on_time (ob_dtl_tenant_mem_manager.cpp:232) [31598][OmtNodeBalancer][T1][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=32] auto free to reserve buffer count(reserve_buffer_min_size=16777216, max_reserve_count=2, buffer_size=65536, ret=0) [2024-08-16 10:20:05.963642] INFO [SQL.DTL] clean_on_timeout (ob_dtl_fc_server.cpp:131) [31598][OmtNodeBalancer][T1][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=18] tenant dfc status(ret=0, get_tenant_id()=1, get_channel_cnt()=0, get_current_buffer_used()=0, get_current_blocked_cnt()=0, get_current_buffer_cnt()=0, get_max_parallel()=20, get_max_blocked_buffer_size()=6553600, get_max_buffer_size()=13107200, get_accumulated_blocked_cnt()=11, get_max_size_per_channel()=655360) [2024-08-16 10:20:05.973412] INFO [SHARE] refresh_resource_plan (ob_resource_plan_manager.cpp:122) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=11] refresh resource plan success(tenant_id=1001, plan_name=, directives=[]) [2024-08-16 10:20:05.976157] INFO [SHARE] refresh_resource_user_mapping_rule (ob_resource_mapping_rule_manager.cpp:159) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=22] refresh resource user mapping rule(tenant_id=1001, plan=, user_rules=[]) [2024-08-16 10:20:05.978632] INFO [SHARE] refresh_resource_function_mapping_rule (ob_resource_mapping_rule_manager.cpp:196) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=36] refresh_resource_function_mapping_rule(tenant_id=1001, plan=, rules=[]) [2024-08-16 10:20:05.978656] INFO [SHARE] refresh_resource_mapping_rule (ob_resource_mapping_rule_manager.cpp:100) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=23] refresh resource mapping rule success(tenant_id=1001, plan=) [2024-08-16 10:20:05.983846] INFO [SQL.DTL] calc_max_buffer (ob_dtl_fc_server.cpp:159) [31598][OmtNodeBalancer][T1001][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=7] trace tenant dfc parameters(max_parallel_cnt_=10, max_blocked_buffer_size_=3276800, max_buffer_size_=6553600) [2024-08-16 10:20:05.983905] INFO [SQL.DTL] auto_free_on_time (ob_dtl_tenant_mem_manager.cpp:232) [31598][OmtNodeBalancer][T1001][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=23] auto free to reserve buffer count(reserve_buffer_min_size=16777216, max_reserve_count=2, buffer_size=65536, ret=0) [2024-08-16 10:20:05.983924] INFO [SQL.DTL] clean_on_timeout (ob_dtl_fc_server.cpp:131) [31598][OmtNodeBalancer][T1001][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=16] tenant dfc status(ret=0, get_tenant_id()=1001, get_channel_cnt()=0, get_current_buffer_used()=0, get_current_blocked_cnt()=0, get_current_buffer_cnt()=0, get_max_parallel()=10, get_max_blocked_buffer_size()=3276800, get_max_buffer_size()=6553600, get_accumulated_blocked_cnt()=0, get_max_size_per_channel()=655360) [2024-08-16 10:20:05.988144] INFO [SHARE] refresh_resource_plan (ob_resource_plan_manager.cpp:122) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=11] refresh resource plan success(tenant_id=1002, plan_name=, directives=[]) [2024-08-16 10:20:05.990230] INFO [SHARE] refresh_resource_user_mapping_rule (ob_resource_mapping_rule_manager.cpp:159) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=22] refresh resource user mapping rule(tenant_id=1002, plan=, user_rules=[]) [2024-08-16 10:20:05.992355] INFO [SHARE] refresh_resource_function_mapping_rule (ob_resource_mapping_rule_manager.cpp:196) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=22] refresh_resource_function_mapping_rule(tenant_id=1002, plan=, rules=[]) [2024-08-16 10:20:05.992377] INFO [SHARE] refresh_resource_mapping_rule (ob_resource_mapping_rule_manager.cpp:100) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=21] refresh resource mapping rule success(tenant_id=1002, plan=) [2024-08-16 10:20:05.993020] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [31597][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=524] dump tenant info(tenant={id:1, tenant_meta:{unit:{tenant_id:1, unit_id:1000, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1000, name:"hidden_sys_unit", resource:{min_cpu:2, max_cpu:2, memory_size:"1GB", log_disk_size:"0GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:2}}, mode:0, create_timestamp:1722932515520653, is_removed:false}, super_block:{tenant_id:1, replay_start_point:ObLogCursor{file_id=2, log_id=5654, offset=657873}, ls_meta_entry:[-1](ver=0,mode=0,seq=0), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:true, version:2}, create_status:1}, unit_min_cpu:"2.000000000000000000e+00", unit_max_cpu:"2.000000000000000000e+00", total_worker_cnt:38, min_worker_cnt:22, max_worker_cnt:150, stopped:0, worker_us:236156242631, recv_hp_rpc_cnt:2505785, recv_np_rpc_cnt:1588831, recv_lp_rpc_cnt:0, recv_mysql_cnt:14351719, recv_task_cnt:796866, recv_large_req_cnt:0, tt_large_quries:1456, pop_normal_cnt:6817907133, workers:22, nesting workers:8, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=456 cnt[3]=0 cnt[4]=0 cnt[5]=61103954 cnt[6]=13 cnt[7]=2 cnt[8]=0 cnt[9]=0 , group_map:group_id = 2,queue_size = 0,recv_req_cnt = 1829,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 2,token_change = 1722932514913013 group_id = 10,queue_size = 0,recv_req_cnt = 26884383,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 2,token_change = 1722932514916132 group_id = 15,queue_size = 0,recv_req_cnt = 5403,min_worker_cnt = 2,max_worker_cnt = 2,worker_cnt = 0,token_change = 1723771320518099 group_id = 5,queue_size = 0,recv_req_cnt = 64,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 2,token_change = 1722932514916435 group_id = 19,queue_size = 0,recv_req_cnt = 5,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 0,token_change = 1722932502279864 group_id = 100,queue_size = 0,recv_req_cnt = 275,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 0,token_change = 1723726142832261 group_id = 9,queue_size = 0,recv_req_cnt = 14309,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 2,token_change = 1722932514916640 group_id = 17,queue_size = 0,recv_req_cnt = 1151,min_worker_cnt = 2,max_worker_cnt = 150,worker_cnt = 0,token_change = 1722931691277014 group_id = 1,queue_size = 0,recv_req_cnt = 59213,min_worker_cnt = 8,max_worker_cnt = 8,worker_cnt = 0,token_change = 1722932561115577 , rpc_stat_info: pcode=0x4a9:cnt=90 pcode=0x51f:cnt=43 pcode=0x152:cnt=5 pcode=0x518:cnt=2, token_change_ts:1722932514916861, tenant_role:1}) [2024-08-16 10:20:05.993778] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [31597][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=756] dump tenant info(tenant={id:508, tenant_meta:{unit:{tenant_id:508, unit_id:1000, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1000, name:"virtual_tenant_unit", resource:{min_cpu:5, max_cpu:5, memory_size:"1GB", log_disk_size:"2GB", min_iops:10000, max_iops:50000, iops_weight:0}}, mode:0, create_timestamp:1720514721373193, is_removed:false}, super_block:{tenant_id:508, replay_start_point:ObLogCursor{file_id=1, log_id=1, offset=0}, ls_meta_entry:[-1](ver=0,mode=0,seq=0), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:true, version:2}, create_status:1}, unit_min_cpu:"5.000000000000000000e+00", unit_max_cpu:"5.000000000000000000e+00", total_worker_cnt:52, min_worker_cnt:52, max_worker_cnt:150, stopped:0, worker_us:47267989371, recv_hp_rpc_cnt:0, recv_np_rpc_cnt:20599088, recv_lp_rpc_cnt:0, recv_mysql_cnt:0, recv_task_cnt:0, recv_large_req_cnt:0, tt_large_quries:0, pop_normal_cnt:16195631074, workers:52, nesting workers:0, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=0 cnt[3]=0 cnt[4]=0 cnt[5]=0 cnt[6]=0 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 15,queue_size = 0,recv_req_cnt = 26,min_worker_cnt = 2,max_worker_cnt = 2,worker_cnt = 0,token_change = 1723040524903572 , rpc_stat_info: pcode=0x5c0:cnt=85, token_change_ts:1720514721381040, tenant_role:0}) [2024-08-16 10:20:05.994571] INFO [SQL.DTL] calc_max_buffer (ob_dtl_fc_server.cpp:159) [31598][OmtNodeBalancer][T1002][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=8] trace tenant dfc parameters(max_parallel_cnt_=80, max_blocked_buffer_size_=26214400, max_buffer_size_=52428800) [2024-08-16 10:20:05.994628] INFO [SQL.DTL] auto_free_on_time (ob_dtl_tenant_mem_manager.cpp:232) [31598][OmtNodeBalancer][T1002][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=22] auto free to reserve buffer count(reserve_buffer_min_size=16777216, max_reserve_count=2, buffer_size=65536, ret=0) [2024-08-16 10:20:05.994644] INFO [SQL.DTL] clean_on_timeout (ob_dtl_fc_server.cpp:131) [31598][OmtNodeBalancer][T1002][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=15] tenant dfc status(ret=0, get_tenant_id()=1002, get_channel_cnt()=0, get_current_buffer_used()=0, get_current_blocked_cnt()=0, get_current_buffer_cnt()=0, get_max_parallel()=80, get_max_blocked_buffer_size()=26214400, get_max_buffer_size()=52428800, get_accumulated_blocked_cnt()=21568, get_max_size_per_channel()=655360) [2024-08-16 10:20:05.994665] INFO [SERVER.OMT] run1 (ob_tenant_node_balancer.cpp:109) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=9] refresh tenant units(sys_unit_cnt=3, units=[{tenant_id:1002, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:8, max_cpu:8, memory_size:"6GB", log_disk_size:"18.9GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}, {tenant_id:1001, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"2.1GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}], ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:05.994708] INFO set_nway (ob_tenant_mutil_allocator.cpp:335) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=35] finish set nway(tenant_id_=1002, nway=8) [2024-08-16 10:20:05.994718] INFO update_tenant_mem_limit (ob_tenant_mutil_allocator_mgr.cpp:380) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=9] ObTenantMutilAllocator update tenant mem_limit finished(ret=0, tenant_id=1002, nway=8, new_tma_limit=3221225450, pre_tma_limit=3221225450, cur_memstore_limit_percent=50, tenant_config={tenant_id:1002, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:8, max_cpu:8, memory_size:"6GB", log_disk_size:"18.9GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}) [2024-08-16 10:20:05.994737] INFO update_tenant_mem_limit (ob_tenant_mutil_allocator_mgr.cpp:391) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=15] succ to set_memstore_threshold of memstore allocator(tenant_id=1002, ret=0) [2024-08-16 10:20:05.994742] INFO set_nway (ob_tenant_mutil_allocator.cpp:335) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=4] finish set nway(tenant_id_=1001, nway=1) [2024-08-16 10:20:05.994746] INFO update_tenant_mem_limit (ob_tenant_mutil_allocator_mgr.cpp:380) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=3] ObTenantMutilAllocator update tenant mem_limit finished(ret=0, tenant_id=1001, nway=1, new_tma_limit=536870900, pre_tma_limit=536870900, cur_memstore_limit_percent=50, tenant_config={tenant_id:1001, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"2.1GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}) [2024-08-16 10:20:05.994756] INFO update_tenant_mem_limit (ob_tenant_mutil_allocator_mgr.cpp:391) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=8] succ to set_memstore_threshold of memstore allocator(tenant_id=1001, ret=0) [2024-08-16 10:20:05.994462] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [31597][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=683] dump tenant info(tenant={id:1001, tenant_meta:{unit:{tenant_id:1001, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"2.1GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}, super_block:{tenant_id:1001, replay_start_point:ObLogCursor{file_id=1, log_id=1, offset=0}, ls_meta_entry:[-1](ver=0,mode=0,seq=0), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:false, version:2}, create_status:1}, unit_min_cpu:"1.000000000000000000e+00", unit_max_cpu:"1.000000000000000000e+00", total_worker_cnt:7, min_worker_cnt:6, max_worker_cnt:150, stopped:0, worker_us:7614885058, recv_hp_rpc_cnt:14124993, recv_np_rpc_cnt:3166730, recv_lp_rpc_cnt:0, recv_mysql_cnt:0, recv_task_cnt:0, recv_large_req_cnt:0, tt_large_quries:4775, pop_normal_cnt:1317880328, workers:6, nesting workers:0, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=0 cnt[3]=0 cnt[4]=0 cnt[5]=0 cnt[6]=0 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 5,queue_size = 0,recv_req_cnt = 78,min_worker_cnt = 1,max_worker_cnt = 150,worker_cnt = 1,token_change = 1720909340702942 , rpc_stat_info: pcode=0x743:cnt=20 pcode=0x651:cnt=10 pcode=0x745:cnt=10, token_change_ts:1720516534273175, tenant_role:1}) [2024-08-16 10:20:05.995318] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1AF-0-0] [lt=49][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509910, log_task={header:{begin_lsn:{lsn:18544229571}, end_lsn:{lsn:18544229693}, log_id:53509910, min_scn:{val:1723774805887235000, v:0}, max_scn:{val:1723774805887235000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544229449}, prev_proposal_id:7, committed_end_lsn:{lsn:18544229571}, data_checksum:1913811101, accum_checksum:1545114045, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774805895060, freeze_ts:1723774805895060, submit_ts:1723774805895065, flushed_ts:1723774805895724, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:659}, fs_cb_begin_ts=1723774805995300, log_life_time=100240) [2024-08-16 10:20:05.995119] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [31597][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=655] dump tenant info(tenant={id:1002, tenant_meta:{unit:{tenant_id:1002, unit_id:1014, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1059, name:"config_epean_zone2_C8G7_xkg", resource:{min_cpu:8, max_cpu:8, memory_size:"6GB", log_disk_size:"18.9GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:8}}, mode:0, create_timestamp:1720516533762742, is_removed:false}, super_block:{tenant_id:1002, replay_start_point:ObLogCursor{file_id=65, log_id=18553178, offset=36846227}, ls_meta_entry:[44114](ver=0,mode=0,seq=226903), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:false, version:2}, create_status:1}, unit_min_cpu:"8.000000000000000000e+00", unit_max_cpu:"8.000000000000000000e+00", total_worker_cnt:60, min_worker_cnt:34, max_worker_cnt:150, stopped:0, worker_us:432349133077, recv_hp_rpc_cnt:4509861, recv_np_rpc_cnt:5675644, recv_lp_rpc_cnt:0, recv_mysql_cnt:16147303, recv_task_cnt:159913, recv_large_req_cnt:0, tt_large_quries:27992571, pop_normal_cnt:8315064363, workers:34, nesting workers:8, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=2588102 cnt[3]=0 cnt[4]=0 cnt[5]=5584 cnt[6]=1591 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 2,queue_size = 0,recv_req_cnt = 58190649,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 8,token_change = 1722932655869139 group_id = 15,queue_size = 0,recv_req_cnt = 5401,min_worker_cnt = 2,max_worker_cnt = 2,worker_cnt = 0,token_change = 1723771320558792 group_id = 5,queue_size = 0,recv_req_cnt = 67,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 8,token_change = 1722932655869453 group_id = 19,queue_size = 0,recv_req_cnt = 698941,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 1,token_change = 1723774783862238 group_id = 100,queue_size = 0,recv_req_cnt = 11824,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 0,token_change = 1723774359377709 group_id = 17,queue_size = 0,recv_req_cnt = 8013,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 0,token_change = 1723452012952496 group_id = 16,queue_size = 0,recv_req_cnt = 425,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 0,token_change = 1723646582548182 group_id = 1,queue_size = 0,recv_req_cnt = 327999673,min_worker_cnt = 8,max_worker_cnt = 8,worker_cnt = 1,token_change = 1723774780652594 , rpc_stat_info: pcode=0x1501:cnt=303 pcode=0x1502:cnt=202 pcode=0x1516:cnt=201 pcode=0x150a:cnt=60 pcode=0x150b:cnt=60, token_change_ts:1722932655869739, tenant_role:1}) [2024-08-16 10:20:05.996554] INFO [SERVER] cal_all_part_disk_default_percentage (ob_server_utils.cpp:325) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=3] cal_all_part_disk_default_percentage succ(data_dir="/home/admin/oceanbase/store/ep_test_cluster/sstable", clog_dir="/home/admin/oceanbase/store/ep_test_cluster/clog", shared_mode=true, data_disk_total_size=258645426176, data_disk_default_percentage=60, clog_disk_total_size=258645426176, clog_disk_default_percentage=30) [2024-08-16 10:20:05.996580] INFO [SERVER] decide_disk_size (ob_server_utils.cpp:361) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=24] decide disk size finished(suggested_disk_size=0, suggested_disk_percentage=0, default_disk_percentage=30, total_space=258645426176, disk_size=77593627852) [2024-08-16 10:20:05.996589] INFO [SERVER] get_log_disk_info_in_config (ob_server_utils.cpp:88) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=8] get_log_disk_info_in_config(suggested_data_disk_size=0, suggested_clog_disk_size=0, suggested_data_disk_percentage=0, suggested_clog_disk_percentage=0, log_disk_size=77593627852, log_disk_percentage=0, total_log_disk_size=258645426176) [2024-08-16 10:20:05.996598] INFO [CLOG] try_resize (ob_server_log_block_mgr.cpp:785) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=6] try_resize success(ret=0, log_disk_size=77593627852, total_log_disk_size=258645426176, this={dir::"/home/admin/oceanbase/store/ep_test_cluster/clog/log_pool", dir_fd:14, meta_fd:15, log_pool_meta:{curr_total_size:77577846784, next_total_size:77577846784, status:0}, min_block_id:1108, max_block_id:2020, min_log_disk_size_for_all_tenants_:22548578304, is_inited:true}) [2024-08-16 10:20:05.996619] INFO [SERVER.OMT] run1 (ob_tenant_node_balancer.cpp:133) [31598][OmtNodeBalancer][T0][YB42C0A80080-00061CCC8CC6C26F-0-0] [lt=20] refresh tenant config(tenants=[1, 1001, 1002], ret=0) [2024-08-16 10:20:05.996788] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=8] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:05.996819] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=23] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:05.996960] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:05.996996] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=28][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:05.997023] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:05.997038] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=13][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:06.004379] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.004401] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:06.004410] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.004418] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.004431] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.012664] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=19] gc stale ls task succ [2024-08-16 10:20:06.012787] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=8] gc stale ls task succ [2024-08-16 10:20:06.012817] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=20] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:06.012833] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=11] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:06.012838] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=4] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:06.016233] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:06.017478] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:06.029747] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C23-0-0] [lt=113] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774806029712) [2024-08-16 10:20:06.039882] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=30] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=7) [2024-08-16 10:20:06.048024] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:06.048093] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=37] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:06.048103] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:06.048110] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:06.049939] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=26][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:06.055615] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.055639] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:06.055650] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.055660] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.055688] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.056739] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=42] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:06.058468] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73653-0-0] [lt=25] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=9, avg_log_submit_to_flush_time=634, avg_log_submit_to_slide_time=1059) [2024-08-16 10:20:06.065390] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:06.065414] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:06.080372] INFO [STORAGE] runTimerTask (ob_tenant_tablet_scheduler.cpp:192) [31785][T1_InfoPoolResi][T1][Y0-0000000000000000-0-0] [lt=28] InfoPoolResizeTask(cost_ts=12) [2024-08-16 10:20:06.083450] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1723774806083447, bytes: 1496437411480, bw: 0.125669 MB/s, add_ts: 1003555, add_bytes: 132242 [2024-08-16 10:20:06.083501] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD865C-0-0] [lt=22] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:06.084978] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74446-0-0] [lt=16] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=9, avg_log_submit_to_flush_time=701, avg_log_submit_to_slide_time=1136) [2024-08-16 10:20:06.085016] INFO [PALF] read (log_cache.cpp:100) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1667, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:06.085140] INFO [LIB] stat (utility.h:1197) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=39, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:06.085194] INFO [LIB] stat (utility.h:1197) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=53] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=16, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:06.102644] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1723774806102638, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007641, add_bytes: 0 [2024-08-16 10:20:06.104472] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.104499] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:20:06.104513] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.104525] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.104549] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.107736] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80085-00061CCD1E636C14-0-0] [lt=22][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:06.109206] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:06.109465] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash time detail, (compute_wash_size_time=122, refresh_score_time=68, wash_time=16) [2024-08-16 10:20:06.112879] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=93, clean_time=0, clean_start_pos=188742, clean_num=62914) [2024-08-16 10:20:06.145970] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7E0-0-0] [lt=68][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:06.146594] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [2553][T1002_ObXAHbWor][T1002][Y0-0000000000000000-0-0] [lt=21] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-08-16 10:20:06.146625] INFO [STORAGE.TRANS] run1 (ob_xa_trans_heartbeat_worker.cpp:83) [2553][T1002_ObXAHbWor][T1002][Y0-0000000000000000-0-0] [lt=29] XA scheduler heartbeat task statistics(avg_time=33) [2024-08-16 10:20:06.146604] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7E0-0-0] [lt=27] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:06.148095] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:06.148195] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=11, avg_log_batch_cnt=1, total_group_log_size=1332, avg_group_log_size=121) [2024-08-16 10:20:06.148220] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1332) [2024-08-16 10:20:06.148232] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:06.148238] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:06.148252] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:06.149008] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1210) [2024-08-16 10:20:06.150019] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:134) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=32] ObStandbyTimestampService state(*this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:20:06.150045] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=24][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:06.150294] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2747][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=83] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:20:06.155685] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.155707] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:06.155716] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.155725] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.155739] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.157344] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B29-0-0] [lt=33] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:06.157374] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B29-0-0] [lt=27] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:06.157928] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=29] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=6759242, this={dio_aligned_buf:{buf_write_offset:964, buf_padding_size:3132, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:44277623, truncate_used_ts:10955939}, log_block_size:67108864, dir_fd:203, io_fd:100}, aligned_buf_len=4096, aligned_buf="", aligned_block_offset=6758400, buf_len=122, total_write_size_=12294, total_write_size_after_dio_=425984, ob_pwrite_used_ts=62965, count=101, each_pwrite_cost=623) [2024-08-16 10:20:06.157963] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:06.158469] INFO [PALF] read (log_cache.cpp:100) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=35] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:06.164253] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=17] ====== tenant freeze timer task ====== [2024-08-16 10:20:06.183949] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=37] gc stale ls task succ [2024-08-16 10:20:06.195412] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E2B-0-0] [lt=28][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=9, len=124, pos=98, bitmap=NULL) [2024-08-16 10:20:06.195468] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1BB-0-0] [lt=10][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509912, log_task={header:{begin_lsn:{lsn:18544229815}, end_lsn:{lsn:18544229937}, log_id:53509912, min_scn:{val:1723774805887235002, v:0}, max_scn:{val:1723774805887235002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544229693}, prev_proposal_id:7, committed_end_lsn:{lsn:18544229815}, data_checksum:1913811101, accum_checksum:2462792808, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774806095435, freeze_ts:1723774806095435, submit_ts:1723774806095448, flushed_ts:1723774806096281, gen_to_freeze cost time:0, gen_to_submit cost time:13, submit_to_flush cost time:833}, fs_cb_begin_ts=1723774806195448, log_life_time=100013) [2024-08-16 10:20:06.195853] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E2B-0-0] [lt=28][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=8, len=105, pos=86, bitmap=NULL) [2024-08-16 10:20:06.204518] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.204552] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=30] handle task succ(id={id:1260}) [2024-08-16 10:20:06.204567] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.204581] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.204638] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.231742] INFO [CLOG] is_in_sync (ob_log_handler.cpp:546) [31817][T1_L5_G0][T1002][YB42C0A8007E-00061EFF757F6EAF-0-0] [lt=23] is_in_sync(ret=0, id=1079, is_log_sync=true, is_need_rebuild=false) [2024-08-16 10:20:06.231843] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007E-00061EFF757F6EAF-0-0] [lt=26][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:06.236563] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:06.236585] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=19] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:06.236878] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=19] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:06.237039] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=22] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:06.237111] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=20] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:06.237362] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:06.237741] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=26] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:06.240005] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=34] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=6) [2024-08-16 10:20:06.245146] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AB3-0-0] [lt=19] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:06.245195] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AB3-0-0] [lt=27] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774805988293002, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774805988293002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:06.246129] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80086-00061EFF757FAF14-0-0] [lt=24][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:06.246990] INFO [PALF] runTimerTask (block_gc_timer_task.cpp:101) [2433][T1002_PalfGC][T1002][Y0-0000000000000000-0-0] [lt=42] BlockGCTimerTask success(ret=0, cost_time_us=20, palf_env_impl_={IPalfEnvImpl:{IPalfEnvImpl:"Dummy"}, self:"192.168.0.128:2882", log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002", disk_options_wrapper:{disk_opts_for_stopping_writing:{log_disk_size(MB):19353, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):19353, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):629, sequence:4}, log_alloc_mgr_:{flying_log_task:0, flying_meta_task:0}}) [2024-08-16 10:20:06.248164] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:06.248297] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:06.248316] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:06.248324] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:06.250100] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:06.250641] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [31653][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=26] LogLoopThread round_cost_time(us)(round_cost_time=2) [2024-08-16 10:20:06.255757] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.255777] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:06.255788] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.255798] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.255812] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.264192] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=35] re_register_parent reason(palf_id=1262, self="192.168.0.128:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1723774802261226, last_first_register_time_us=1720517744275280, register_time_us=1723774802261226, curr_time_us=1723774806264189) [2024-08-16 10:20:06.264269] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=32] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1262, self="192.168.0.128:2882", reason=1, leader="192.168.0.133:2882", register_time_us=1723774806264242, last_submit_register_req_time_us=1723774806264242, register_parent_reason=1) [2024-08-16 10:20:06.264296] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=25] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1262, self="192.168.0.128:2882") [2024-08-16 10:20:06.264327] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=9] re_register_parent reason(palf_id=1261, self="192.168.0.128:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1723774802261311, last_first_register_time_us=1720517744277428, register_time_us=1723774802261311, curr_time_us=1723774806264326) [2024-08-16 10:20:06.264366] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=21] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1261, self="192.168.0.128:2882", reason=1, leader="192.168.0.125:2882", register_time_us=1723774806264349, last_submit_register_req_time_us=1723774806264349, register_parent_reason=1) [2024-08-16 10:20:06.264381] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=14] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1261, self="192.168.0.128:2882") [2024-08-16 10:20:06.270333] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=20] current monitor number(seq_=-1) [2024-08-16 10:20:06.283608] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8660-0-0] [lt=65] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:06.284163] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=610, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:06.284184] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:06.284205] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=28, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:06.284250] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=40, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:06.284324] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=10] can close safely, do destroy(*s={this:0x7fa71f01e030, session_id:3225600948, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:true, reader:{fd:129}, err:5, last_decode_time:1723774805756988, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:2084, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-08-16 10:20:06.284365] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=27] unlock session of tenant(conn.sessid_=3225600948, proxy_sessid=0, conn.tenant_id_=1) [2024-08-16 10:20:06.284405] WDIAG [COMMON] get (ob_link_hashmap.h:313) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=4][errcode=0] uref leak check(node->uref_=1074538690, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:06.284380] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=14] connection close(sessid=3225600948, proxy_sessid=0, tenant_id=1, server_id=16, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCC65F2E23-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:32, pkt_rec[0]:{send:obp_mysql_header_{len_:11, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{receive:obp_mysql_header_{len_:125, rec_:125, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[2]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{send:obp_mysql_header_{pkt_num_:1, seq_:3}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[4]:{send:obp_mysql_header_{len_:9, seq_:4}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_mysql_header_{pkt_num_:1, seq_:5}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[6]:{send:obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[7]:{receive:obp_mysql_header_{len_:399, rec_:399, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[8]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[9]:{send:obp_mysql_header_{pkt_num_:2, seq_:4}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[10]:{send:obp_mysql_header_{len_:9, seq_:5}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[11]:{send:obp_mysql_header_{pkt_num_:1, seq_:6}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[12]:{send:obp_mysql_header_{len_:9, seq_:7}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[13]:{receive:obp_mysql_header_{len_:404, rec_:404, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[14]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[15]:{send:obp_mysql_header_{pkt_num_:2, seq_:4}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[16]:{send:obp_mysql_header_{len_:9, seq_:5}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[17]:{send:obp_mysql_header_{pkt_num_:1, seq_:6}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[18]:{send:obp_mysql_header_{len_:9, seq_:7}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[19]:{receive:obp_mysql_header_{len_:755, rec_:755, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[20]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[21]:{send:obp_mysql_header_{pkt_num_:32, seq_:34}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[22]:{send:obp_mysql_header_{len_:9, seq_:35}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[23]:{send:obp_mysql_header_{pkt_num_:4, seq_:39}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[24]:{send:obp_mysql_header_{len_:9, seq_:40}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[25]:{receive:obp_mysql_header_{len_:275, rec_:275, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[26]:{send:obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[27]:{send:obp_mysql_header_{pkt_num_:8, seq_:10}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[28]:{send:obp_mysql_header_{len_:9, seq_:11}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[29]:{send:obp_mysql_header_{pkt_num_:18, seq_:29}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[30]:{send:obp_mysql_header_{len_:9, seq_:30}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[31]:{receive:obp_mysql_header_{len_:1, rec_:1, seq_:0}, pkt_name:"COM_QUIT", obp_mysql_header_.type_:1, obp_mysql_header_.is_send_:0}], disconnect_state=3) [2024-08-16 10:20:06.284489] INFO [SERVER] run (obmp_disconnect.cpp:82) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=21] free session successfully(sessid=3225600948) [2024-08-16 10:20:06.284502] INFO [SERVER] run (obmp_disconnect.cpp:89) [21123][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=12] mark session id unused(sessid=3225600948) [2024-08-16 10:20:06.284642] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7444C-0-0] [lt=21] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=14, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:06.284659] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7444C-0-0] [lt=16] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1156, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:06.290338] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E095C-0-0] [lt=24] current clock is (clock_=0, max_seq_=0, mem_limit=1643328294, cur_mem_hold=289406976, attr_.tenant_id_=1002) [2024-08-16 10:20:06.294148] INFO [STORAGE] runTimerTask (ob_tenant_memory_printer.cpp:33) [31393][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=12] === Run print tenant memory usage task === [2024-08-16 10:20:06.294392] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [31393][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=24] [MEMORY] tenant: 500, limit: 9,223,372,036,854,775,807 hold: 946,462,720 rpc_hold: 0 cache_hold: 0 cache_used: 0 cache_item_count: 0 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 316,022,784 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= GLIBC hold_bytes= 27,262,976 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 115,343,360 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= LIBEASY hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= LOGGER_CTX_ID hold_bytes= 12,582,912 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= RPC_CTX_ID hold_bytes= 6,291,456 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PKT_NIO hold_bytes= 12,640,256 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= SCHEMA_SERVICE hold_bytes= 120,344,576 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= UNEXPECTED_IN_500 hold_bytes= 331,780,096 limit= 9,223,372,036,854,775,807 [2024-08-16 10:20:06.294765] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [31393][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=20] [MEMORY] tenant_id= 500 ctx_id= DEFAULT_CTX_ID hold= 316,022,784 used= 285,589,616 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 52,268,992 used= 50,996,736 count= 3,317 avg_used= 15,374 block_cnt= 3,317 chunk_cnt= 31 mod=TZInfoArray [MEMORY] hold= 33,574,912 used= 33,554,464 count= 1 avg_used= 33,554,464 block_cnt= 1 chunk_cnt= 1 mod=BloomFilter [MEMORY] hold= 18,894,848 used= 18,874,392 count= 1 avg_used= 18,874,392 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDTLINT [MEMORY] hold= 14,376,960 used= 12,837,240 count= 195 avg_used= 65,832 block_cnt= 195 chunk_cnt= 26 mod=[T]ObSessionDIB [MEMORY] hold= 10,907,648 used= 10,866,720 count= 13 avg_used= 835,901 block_cnt= 13 chunk_cnt= 9 mod=IoControl [MEMORY] hold= 9,457,664 used= 9,437,784 count= 1 avg_used= 9,437,784 block_cnt= 1 chunk_cnt= 1 mod=HashBuckInteChe [MEMORY] hold= 8,585,216 used= 8,566,048 count= 1 avg_used= 8,566,048 block_cnt= 1 chunk_cnt= 1 mod=MemDumpContext [MEMORY] hold= 6,865,456 used= 6,785,296 count= 48 avg_used= 141,360 block_cnt= 25 chunk_cnt= 11 mod=PartitTableTask [MEMORY] hold= 5,087,232 used= 5,027,008 count= 6 avg_used= 837,834 block_cnt= 6 chunk_cnt= 4 mod=KvstCachWashStr [MEMORY] hold= 4,746,944 used= 4,726,520 count= 2 avg_used= 2,363,260 block_cnt= 2 chunk_cnt= 2 mod=HashBucTenComMo [MEMORY] hold= 4,739,072 used= 4,718,712 count= 1 avg_used= 4,718,712 block_cnt= 1 chunk_cnt= 1 mod=HashPxBlooFilKe [MEMORY] hold= 4,739,072 used= 4,718,712 count= 1 avg_used= 4,718,712 block_cnt= 1 chunk_cnt= 1 mod=PxP2PDhMgrKey [MEMORY] hold= 4,472,832 used= 3,915,776 count= 68 avg_used= 57,584 block_cnt= 68 chunk_cnt= 28 mod=[T]char [MEMORY] hold= 4,307,328 used= 4,278,711 count= 3 avg_used= 1,426,237 block_cnt= 3 chunk_cnt= 3 mod=SqlDtlMgr [MEMORY] hold= 4,231,936 used= 4,203,008 count= 6 avg_used= 700,501 block_cnt= 5 chunk_cnt= 5 mod=BaseLogWriter [MEMORY] hold= 4,214,880 used= 4,194,328 count= 2 avg_used= 2,097,164 block_cnt= 2 chunk_cnt= 2 mod=SerFuncRegHT [MEMORY] hold= 4,046,848 used= 3,969,504 count= 13 avg_used= 305,346 block_cnt= 13 chunk_cnt= 8 mod=DedupQueue [MEMORY] hold= 3,984,128 used= 3,952,128 count= 498 avg_used= 7,936 block_cnt= 498 chunk_cnt= 30 mod=ObTabletLSCache [MEMORY] hold= 3,353,392 used= 3,282,840 count= 640 avg_used= 5,129 block_cnt= 204 chunk_cnt= 40 mod=DeadLock [MEMORY] hold= 2,621,440 used= 2,619,392 count= 16 avg_used= 163,712 block_cnt= 16 chunk_cnt= 9 mod=Iteratorget_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=1, size_used=5271, mem_used=39515136) [2024-08-16 10:20:06.304591] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.304614] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:06.304626] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.304636] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.304652] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.309140] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=22] ====== tenant freeze timer task ====== [2024-08-16 10:20:06.309185] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=27][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:06.313437] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=50] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:06.313725] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=30] Wash time detail, (compute_wash_size_time=153, refresh_score_time=86, wash_time=16) [2024-08-16 10:20:06.317144] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Cache wash clean map node details(ret=0, clean_node_count=96, clean_time=0, clean_start_pos=251656, clean_num=62914) [2024-08-16 10:20:06.348245] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:06.348286] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=34] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:06.348296] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:06.348302] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:06.350191] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:06.354757] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE85-0-0] [lt=26] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:06.354802] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE85-0-0] [lt=24] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774806003436002, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774806003436002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:06.355897] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.355943] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=28] handle task succ(id={id:1260}) [2024-08-16 10:20:06.355958] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.355984] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=24] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.356009] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.374445] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [2798][T1001_TxLoopWor][T1001][Y0-0000000000000000-0-0] [lt=7] tx gc loop thread is running(MTL_ID()=1001) [2024-08-16 10:20:06.374479] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [2798][T1001_TxLoopWor][T1001][Y0-0000000000000000-0-0] [lt=31] try gc retain ctx [2024-08-16 10:20:06.384710] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=21] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:06.394583] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=26] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574644, offset=42148317}, end_cursor:ObLogCursor{file_id=65, log_id=18574645, offset=42148558}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751758050, len:3619, log_data_len:241, seq:18574644, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:06.395204] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=36] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574645, offset=42148558}, end_cursor:ObLogCursor{file_id=65, log_id=18574646, offset=42148807}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516b0050, len:3378, log_data_len:249, seq:18574645, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:06.395586] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1C7-0-0] [lt=59] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:06.396876] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA8-0-0] [lt=34] push gts task success(*task={:""}) [2024-08-16 10:20:06.403434] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA8-0-0] [lt=25] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:06.404667] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.404693] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:20:06.404708] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.404718] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.404733] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.405788] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=31] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574646, offset=42148807}, end_cursor:ObLogCursor{file_id=65, log_id=18574647, offset=42149056}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751718050, len:3129, log_data_len:249, seq:18574646, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:06.406246] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=38] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574647, offset=42149056}, end_cursor:ObLogCursor{file_id=65, log_id=18574648, offset=42149305}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751730050, len:2880, log_data_len:249, seq:18574647, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:06.425932] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=28] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:06.426002] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=20] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:06.429603] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=27] gts statistics(tenant_id=1001, gts_rpc_cnt=27, get_gts_cache_cnt=109, get_gts_with_stc_cnt=17, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-08-16 10:20:06.429782] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [31771][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=10] ====== checkpoint timer task ====== [2024-08-16 10:20:06.429808] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [31771][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=20] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:20:06.429872] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=31] gts statistics(tenant_id=1002, gts_rpc_cnt=27, get_gts_cache_cnt=405, get_gts_with_stc_cnt=17, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=12, try_wait_gts_elapse_cnt=0) [2024-08-16 10:20:06.431024] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=9] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1) [2024-08-16 10:20:06.431051] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=22] start lib cache evict(tenant_id=1, mem_hold=46137344, mem_limit=53687090, cache_obj_num=123, cache_node_num=123) [2024-08-16 10:20:06.431064] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=11] end lib cache evict(tenant_id=1, cache_evict_num=0, mem_hold=46137344, mem_limit=53687090, cache_obj_num=123, cache_node_num=123) [2024-08-16 10:20:06.431095] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=28] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:06.434051] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=9] Dumping All Cache Objs(alloc_obj_list.count()=123, alloc_obj_list=[{obj_id:27, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190650}, {obj_id:95, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:96, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:174, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:984, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205451}, {obj_id:1112, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:310633}, {obj_id:1476, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179375}, {obj_id:1489, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191695}, {obj_id:1687, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198485}, {obj_id:1732, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172160}, {obj_id:1736, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174033}, {obj_id:1741, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175220}, {obj_id:1758, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205820}, {obj_id:1862, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188466}, {obj_id:2017, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:2151, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189565}, {obj_id:2278, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:2279, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189832}, {obj_id:2280, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176478}, {obj_id:2281, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177168}, {obj_id:2282, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174044}, {obj_id:2283, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174081}, {obj_id:2284, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174036}, {obj_id:2285, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185923}, {obj_id:2286, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220611}, {obj_id:2287, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207401}, {obj_id:2288, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180621}, {obj_id:2289, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177481}, {obj_id:2290, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:2291, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183781}, {obj_id:2292, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2293, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188441}, {obj_id:2294, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260630}, {obj_id:2295, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223446}, {obj_id:2296, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173811}, {obj_id:2297, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172864}, {obj_id:2298, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179482}, {obj_id:2299, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186746}, {obj_id:2300, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204032}, {obj_id:2301, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188714}, {obj_id:2302, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183948}, {obj_id:2303, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182890}, {obj_id:2304, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2305, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:2306, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2307, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:2308, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2309, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:2310, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2311, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:2312, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203568}, {obj_id:2313, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191582}, {obj_id:2314, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182638}, {obj_id:2315, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182415}, {obj_id:2316, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184713}, {obj_id:2317, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182209}, {obj_id:2318, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205928}, {obj_id:2319, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223722}, {obj_id:2320, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238225}, {obj_id:2321, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194888}, {obj_id:2322, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220593}, {obj_id:2323, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188416}, {obj_id:2325, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188394}, {obj_id:2327, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196424}, {obj_id:2330, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229007}, {obj_id:2333, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194173}, {obj_id:2336, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210443}, {obj_id:2337, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206506}, {obj_id:2338, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180446}, {obj_id:2339, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234757}, {obj_id:2340, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179743}, {obj_id:2341, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179601}, {obj_id:2342, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184831}, {obj_id:2343, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188563}, {obj_id:2344, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237176}, {obj_id:2345, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198348}, {obj_id:2346, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256258}, {obj_id:2347, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185263}, {obj_id:2348, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196309}, {obj_id:2349, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:265571}, {obj_id:2350, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260809}, {obj_id:2351, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256181}, {obj_id:2352, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299476}, {obj_id:2353, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:270176}, {obj_id:2354, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:305943}, {obj_id:2355, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172528}, {obj_id:2356, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176862}, {obj_id:2357, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183129}, {obj_id:2358, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175603}, {obj_id:2359, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181253}, {obj_id:2360, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181039}, {obj_id:2361, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185755}, {obj_id:2362, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202273}, {obj_id:2363, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:267576}, {obj_id:2364, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183894}, {obj_id:2365, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217508}, {obj_id:2366, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211579}, {obj_id:2367, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184855}, {obj_id:2368, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228602}, {obj_id:2369, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217173}, {obj_id:2370, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214453}, {obj_id:2371, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206378}, {obj_id:2372, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178773}, {obj_id:2373, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177464}, {obj_id:2374, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230030}, {obj_id:2375, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177376}, {obj_id:2376, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183047}, {obj_id:2377, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210108}, {obj_id:2378, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202314}, {obj_id:2379, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202343}, {obj_id:2380, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198396}, {obj_id:2381, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229073}, {obj_id:2382, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204285}, {obj_id:2383, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179141}, {obj_id:2384, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2385, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179385}, {obj_id:2386, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180644}, {obj_id:2387, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2388, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175472}, {obj_id:2389, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175472}, {obj_id:2390, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2391, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174540}, {obj_id:2392, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188792}]) [2024-08-16 10:20:06.434304] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [31671][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=237] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:06.436265] INFO [SQL.EXE] run2 (ob_maintain_dependency_info_task.cpp:221) [31493][MaintainDepInfo][T0][Y0-0000000000000000-0-0] [lt=17] [ASYNC TASK QUEUE](queue_size=0) [2024-08-16 10:20:06.440118] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=29] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=5) [2024-08-16 10:20:06.440853] INFO [CLOG] get_replay_process (ob_replay_status.cpp:1073) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=15] replay status is not follower(min_unreplayed_lsn={lsn:8469424174}, base_lsn={lsn:0}, this={ls_id_:{id:1260}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:1, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723617929662670, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:2710542, ret_code_:0}}, next_to_submit_lsn_:{lsn:8469424174}, next_to_submit_scn_:{val:1723617929041268003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516550107534003, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69fc05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8469424052}, end_lsn:{lsn:8469424174}, read_buf:{buf_len_:3682304, buf_:0x7fa69fc05000}, block_size:67104768, log_storage_:0x7fa6bd55b470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723617929041268002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723617929041268002, v:0}, data_checksum:3690972251, flag:0}}, init_mode_version:0, accumulate_checksum:2828504552, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723617929041268002, v:0}}}}}) [2024-08-16 10:20:06.440936] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=82] get_replay_process success(id={id:1260}, replayed_log_size=8469424174, unreplayed_log_size=0) [2024-08-16 10:20:06.440955] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=11] get_replay_process success(id={id:1261}, replayed_log_size=32601570716, unreplayed_log_size=0) [2024-08-16 10:20:06.440966] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=6] get_replay_process success(id={id:1262}, replayed_log_size=8394851512, unreplayed_log_size=0) [2024-08-16 10:20:06.440993] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=15] get_replay_process success(id={id:1079}, replayed_log_size=5525905189, unreplayed_log_size=0) [2024-08-16 10:20:06.441019] INFO [CLOG] runTimerTask (ob_log_replay_service.cpp:159) [2528][T1002_ReplayPro][T1002][Y0-0000000000000000-0-0] [lt=11] dump tenant replay process(tenant_id=1002, unreplayed_log_size(MB)=0, estimate_time(second)=0, replayed_log_size(MB)=52444, last_replayed_log_size(MB)=52444, round_cost_time(second)=10, pending_replay_log_size(MB)=0) [2024-08-16 10:20:06.448287] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:06.448407] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:06.448424] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:06.448432] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:06.449066] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT WRITE LOG COST TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=619, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:06.449123] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=46] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=29, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:06.449173] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT FLUSH CB COST TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:06.449537] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73667-0-0] [lt=23] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=9, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:06.449555] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73667-0-0] [lt=17] [PALF STAT LOG LIFE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=1259, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:06.450271] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=35][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:06.452833] INFO [STORAGE.TRANS] alloc (ob_trans_factory.cpp:141) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E0966-0-0] [lt=35] ObTransCtx statistics(active_coord_ctx_count=0, active_part_ctx_count=1, total_release_part_ctx_count=13) [2024-08-16 10:20:06.452999] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E0966-0-0] [lt=23] current clock is (clock_=0, max_seq_=0, mem_limit=1643328294, cur_mem_hold=289406976, attr_.tenant_id_=1002) [2024-08-16 10:20:06.455931] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.455955] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:06.455965] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.455976] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.455991] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.456009] WDIAG [SERVER] do_process (obmp_query.cpp:1018) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E0968-0-0] [lt=29][errcode=0] need disconnect(ret=0, need_disconnect=true) [2024-08-16 10:20:06.480631] INFO [STORAGE] runTimerTask (ob_tenant_tablet_scheduler.cpp:174) [31784][T1_SSTableGC][T1][YB42C0A80080-00061CCC8F837941-0-0] [lt=43] SSTableGCTask(cost_ts=7) [2024-08-16 10:20:06.483733] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8664-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:06.495580] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1CD-0-0] [lt=25] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1623) [2024-08-16 10:20:06.495623] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1CD-0-0] [lt=28] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=12, avg_log_submit_to_flush_time=696, avg_log_submit_to_slide_time=90989) [2024-08-16 10:20:06.495642] INFO [PALF] read (log_cache.cpp:100) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1623, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:06.504743] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.504765] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:06.504777] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.504787] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.504803] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.517606] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=23] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:06.517875] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=31] Wash time detail, (compute_wash_size_time=117, refresh_score_time=65, wash_time=16) [2024-08-16 10:20:06.521220] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Cache wash clean map node details(ret=0, clean_node_count=86, clean_time=0, clean_start_pos=314570, clean_num=62914) [2024-08-16 10:20:06.526601] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=31] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7023, mem_used=12478464) [2024-08-16 10:20:06.534019] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=33] start do ls ha handler(ls_id_array_=[{id:1261}, {id:1262}, {id:1079}, {id:1260}]) [2024-08-16 10:20:06.541707] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=10] trace max work area(auto_calc=true, tenant_max_memory_limit=1073741824, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=53687091, max_wa_memory_size=53687091, tmp_max_wa_memory_size=53687091, pre_mem_target=53687091, remain_memory_size=53687091, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=332681216, washable_size=-2, max_workarea_memory_size=53687091, max_tenant_memory_size=741060608, wash_ratio=6, tenant_id=1001) [2024-08-16 10:20:06.541790] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=40] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=6710886, calc_global_bound_size=1048576000000, mem_target_=53687091, wa_max_memory_size=53687091, profile_cnt_=0, total_memory_size=0, max_wa_size=53687091, avg_bound_size=6710886, max_bound_size=6710886, min_bound_size_=1048576) [2024-08-16 10:20:06.541822] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=30] timer to calc global bound size(ret=0, global_bound_size_=6710886, manual_calc_cnt_=0, drift_size_=0, pre_drift_size=0, wa_max_memory_size=53687091, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1001, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=6710886, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=53687091, auto_calc=true, sql_mem_callback_.get_total_dump_size()=0) [2024-08-16 10:20:06.541877] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=14] trace max work area(auto_calc=true, tenant_max_memory_limit=6442450944, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=3865470566, max_wa_memory_size=3865470566, tmp_max_wa_memory_size=3865470566, pre_mem_target=3865470566, remain_memory_size=3865470566, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=2461601792, washable_size=-2, max_workarea_memory_size=3865470566, max_tenant_memory_size=3980849152, wash_ratio=6, tenant_id=1002) [2024-08-16 10:20:06.541969] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=30] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=483183820, calc_global_bound_size=1048576000000, mem_target_=3865470566, wa_max_memory_size=3865470566, profile_cnt_=0, total_memory_size=0, max_wa_size=3865470566, avg_bound_size=483183820, max_bound_size=483183820, min_bound_size_=1048576) [2024-08-16 10:20:06.541987] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=17] timer to calc global bound size(ret=0, global_bound_size_=483183820, manual_calc_cnt_=19637, drift_size_=0, pre_drift_size=0, wa_max_memory_size=3865470566, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1002, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=483183820, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=3865470566, auto_calc=true, sql_mem_callback_.get_total_dump_size()=1112786688) [2024-08-16 10:20:06.543658] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [31638][T1_Occam][T1][YB42C0A80080-00061CCCBF6D3E0E-0-0] [lt=21] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:06.543689] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [31638][T1_Occam][T1][YB42C0A80080-00061CCCBF6D3E10-0-0] [lt=22] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:06.548085] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [31787][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=39] try gc retain ctx [2024-08-16 10:20:06.548358] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:06.548384] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:06.548390] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:06.548394] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:06.550345] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:06.556004] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.556025] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:06.556036] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.556045] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.556065] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.556967] INFO [SHARE] refresh_primary_cluster_id (ob_alive_server_tracer.cpp:397) [31391][ServerTracerTim][T0][YB42C0A80080-00061CCC8EABC3A3-0-0] [lt=20] refresh primary cluster finish(ret=0, primary_cluster_id_=-1) [2024-08-16 10:20:06.557619] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B31-0-0] [lt=13] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:06.572842] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [31733][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=37] run rewrite rule refresh task(rule_mgr_->tenant_id_=1) [2024-08-16 10:20:06.575706] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 438.00) [2024-08-16 10:20:06.575733] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=21] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:06.575791] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=25] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.99, rt: 420.75) [2024-08-16 10:20:06.575810] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=14] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:06.575848] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=30] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575864] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575871] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575876] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575882] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575890] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575897] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575902] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575906] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575910] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575914] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575920] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575930] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575935] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575941] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.575950] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:06.583291] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [31738][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=46] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:940, allocator used:3737440, allocator total:3859072}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:140, allocator used:9182740, allocator total:9182740}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=13041812, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:63, allocator used:124992, allocator total:130816}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:06.591582] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=12][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:06.593946] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=37] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:06.594036] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=22][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:06.594059] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=21][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:06.594067] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=7] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:06.595648] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1D1-0-0] [lt=22][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509917, log_task={header:{begin_lsn:{lsn:18544230706}, end_lsn:{lsn:18544230828}, log_id:53509917, min_scn:{val:1723774806387358002, v:0}, max_scn:{val:1723774806387358002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544230303}, prev_proposal_id:7, committed_end_lsn:{lsn:18544230706}, data_checksum:1913811101, accum_checksum:1947343730, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774806495560, freeze_ts:1723774806495560, submit_ts:1723774806495565, flushed_ts:1723774806496123, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:558}, fs_cb_begin_ts=1723774806595636, log_life_time=100076) [2024-08-16 10:20:06.596233] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=21, stat_interval=1000000, avg (count/cost)=34, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:06.604827] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.604852] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:20:06.604863] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.604874] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.604889] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.607674] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=10] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:06.618198] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=25] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60735, mem_used=137263104) [2024-08-16 10:20:06.640220] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=27] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=4) [2024-08-16 10:20:06.645605] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [31772][T1_Flush][T1][Y0-0000000000000000-0-0] [lt=12] ====== traversal_flush timer task ====== [2024-08-16 10:20:06.645639] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:338) [31772][T1_Flush][T1][Y0-0000000000000000-0-0] [lt=26] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:20:06.648423] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:06.648558] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=47] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:06.648580] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:06.648590] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:06.650415] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:06.656109] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=41] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.656131] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:06.656145] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.656160] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.656183] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.658172] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=564, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:06.658279] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=38, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:06.658669] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74458-0-0] [lt=38] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:06.658685] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74458-0-0] [lt=15] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1052, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:06.669420] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=20] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:06.695871] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1D7-0-0] [lt=7][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509918, log_task={header:{begin_lsn:{lsn:18544230828}, end_lsn:{lsn:18544230950}, log_id:53509918, min_scn:{val:1723774806582705000, v:0}, max_scn:{val:1723774806582705000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544230706}, prev_proposal_id:7, committed_end_lsn:{lsn:18544230828}, data_checksum:1913811101, accum_checksum:3133166645, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774806595614, freeze_ts:1723774806595615, submit_ts:1723774806595620, flushed_ts:1723774806596249, gen_to_freeze cost time:1, gen_to_submit cost time:6, submit_to_flush cost time:629}, fs_cb_begin_ts=1723774806695858, log_life_time=100244) [2024-08-16 10:20:06.704904] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.704933] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:20:06.704945] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.704956] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.704971] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.721784] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=15] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:06.722129] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=51] Wash time detail, (compute_wash_size_time=168, refresh_score_time=88, wash_time=20) [2024-08-16 10:20:06.725574] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=24] Cache wash clean map node details(ret=0, clean_node_count=99, clean_time=0, clean_start_pos=377484, clean_num=62914) [2024-08-16 10:20:06.739627] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=8] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1001) [2024-08-16 10:20:06.739671] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=39] start lib cache evict(tenant_id=1001, mem_hold=48234496, mem_limit=53687090, cache_obj_num=115, cache_node_num=115) [2024-08-16 10:20:06.739683] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=10] end lib cache evict(tenant_id=1001, cache_evict_num=0, mem_hold=48234496, mem_limit=53687090, cache_obj_num=115, cache_node_num=115) [2024-08-16 10:20:06.739691] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=6] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:06.742647] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=4] Dumping All Cache Objs(alloc_obj_list.count()=115, alloc_obj_list=[{obj_id:4206, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190714}, {obj_id:6967, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:7888, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190463}, {obj_id:8067, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198580}, {obj_id:8069, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179360}, {obj_id:8462, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:252321}, {obj_id:8631, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176478}, {obj_id:8633, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:8641, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188033}, {obj_id:8645, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8646, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:8647, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8648, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:8649, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8651, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8652, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8653, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188613}, {obj_id:8682, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187574}, {obj_id:8692, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234101}, {obj_id:8697, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227212}, {obj_id:8767, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245433}, {obj_id:8786, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279819}, {obj_id:8795, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:281759}, {obj_id:8805, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191548}, {obj_id:8840, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:293038}, {obj_id:8842, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:324921}, {obj_id:8843, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:456542}, {obj_id:8851, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:575647}, {obj_id:8854, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:347535}, {obj_id:8873, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:452546}, {obj_id:8878, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:582753}, {obj_id:8885, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:474728}, {obj_id:8886, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:329638}, {obj_id:8891, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:452691}, {obj_id:8892, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:375165}, {obj_id:8896, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:356761}, {obj_id:8897, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290927}, {obj_id:8898, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:413784}, {obj_id:8906, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8907, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8908, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189807}, {obj_id:8909, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189568}, {obj_id:8910, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176333}, {obj_id:8912, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196245}, {obj_id:8913, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8914, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180998}, {obj_id:8915, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174064}, {obj_id:8916, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177216}, {obj_id:8917, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:8918, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196176}, {obj_id:8919, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177512}, {obj_id:8920, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8921, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:8922, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:8923, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180749}, {obj_id:8925, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:253523}, {obj_id:8926, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199084}, {obj_id:8927, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178773}, {obj_id:8928, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177464}, {obj_id:8929, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179416}, {obj_id:8930, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180672}, {obj_id:8931, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8932, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186657}, {obj_id:8933, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233944}, {obj_id:8934, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185434}, {obj_id:8935, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198144}, {obj_id:8936, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207413}, {obj_id:8937, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189999}, {obj_id:8938, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188514}, {obj_id:8939, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188880}, {obj_id:8940, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231829}, {obj_id:8941, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190949}, {obj_id:8942, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192036}, {obj_id:8943, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187766}, {obj_id:8944, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195808}, {obj_id:8946, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176485}, {obj_id:8947, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:386890}, {obj_id:8948, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8949, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202973}, {obj_id:8950, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183752}, {obj_id:8951, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223501}, {obj_id:8952, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188501}, {obj_id:8953, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186993}, {obj_id:8954, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:8955, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:8956, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:8957, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176676}, {obj_id:8958, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175405}, {obj_id:8959, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199144}, {obj_id:8960, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205144}, {obj_id:8961, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218139}, {obj_id:8962, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218827}, {obj_id:8963, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217152}, {obj_id:8964, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218674}, {obj_id:8965, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8966, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206801}, {obj_id:8967, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256033}, {obj_id:8968, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8969, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199798}, {obj_id:8970, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199390}, {obj_id:8971, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220936}, {obj_id:8972, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209427}, {obj_id:8973, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220503}, {obj_id:8974, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221262}, {obj_id:8975, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189045}, {obj_id:8976, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194445}, {obj_id:8977, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189493}, {obj_id:8978, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196989}, {obj_id:8979, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190072}, {obj_id:8980, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202973}, {obj_id:8981, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181806}, {obj_id:8982, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180351}, {obj_id:8983, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177512}, {obj_id:8984, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188650}, {obj_id:8985, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179512}]) [2024-08-16 10:20:06.742850] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [2692][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=198] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:06.748496] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:06.748600] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=19, stat_interval=1000000, avg (count/cost)=42, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:06.748649] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT APPEND COST TIME](cur_stat_count=19, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:06.748658] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:06.748665] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:06.748675] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:06.749843] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F73671-0-0] [lt=48] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=19, avg_log_gen_to_freeze_time=4, avg_log_gen_to_submit_time=57, avg_log_submit_to_flush_time=686, avg_log_submit_to_slide_time=1185) [2024-08-16 10:20:06.750485] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:06.756215] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.756236] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:06.756246] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.756256] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.756270] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.796010] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1DD-0-0] [lt=27] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:06.796102] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1DD-0-0] [lt=36] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:06.796122] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1DD-0-0] [lt=19] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=91034, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:06.796801] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1623) [2024-08-16 10:20:06.804978] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.805002] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:06.805014] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.805024] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.805038] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.840327] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=25] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=3) [2024-08-16 10:20:06.848569] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:06.848684] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=24] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:06.848697] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:06.848703] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:06.849331] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=37] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=5756) [2024-08-16 10:20:06.850557] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:06.856284] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.856305] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:06.856315] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.856325] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.856340] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.864172] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [2421][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=48] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,1,False,{False,},False,False,True}, {1024,1,False,{False,},False,False,True}, {1026,1,False,{False,},False,False,True}, {1028,0,False,{False,},False,False,True}, {1079,1,False,{False,},False,False,True}, {1260,0,False,{False,},False,False,True}, {1261,1,False,{False,},False,False,True}, {1262,1,False,{False,},False,False,True}]) [2024-08-16 10:20:06.897186] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=667, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:06.905059] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.905103] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=42] handle task succ(id={id:1260}) [2024-08-16 10:20:06.905114] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.905124] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.905143] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.926019] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:06.926317] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=33] Wash time detail, (compute_wash_size_time=123, refresh_score_time=84, wash_time=16) [2024-08-16 10:20:06.929565] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Cache wash clean map node details(ret=0, clean_node_count=99, clean_time=0, clean_start_pos=440398, clean_num=62914) [2024-08-16 10:20:06.930310] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=22] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:06.933691] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80081-00061EFF86EC941C-0-0] [lt=43][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:06.948641] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:06.948758] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:06.948777] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:06.948784] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:06.950627] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:06.956359] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:06.956380] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:06.956390] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:06.956400] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:06.956415] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:06.956479] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2417][T1002_Occam][T1002][YB42C0A80080-00061CCCBF7D3866-0-0] [lt=20] for each ls(succ_num=4, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:06.992221] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [31728][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=24] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:20:06.997061] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=9] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:06.997117] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=45] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:06.997257] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:06.997278] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=16][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:06.997287] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=5][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:06.997294] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=6][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:07.005125] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.005148] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:07.005163] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.005173] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.005187] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.012769] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=20] gc stale ls task succ [2024-08-16 10:20:07.013002] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=3] gc stale ls task succ [2024-08-16 10:20:07.013030] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=18] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:07.013052] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=16] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:07.013058] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=5] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:07.016104] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:07.017255] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=23] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:07.030568] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C28-0-0] [lt=8] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774807030532) [2024-08-16 10:20:07.040453] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=30] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=2) [2024-08-16 10:20:07.048726] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:07.048841] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:07.048861] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:07.048869] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:07.049682] INFO [STORAGE.DUP_TABLE] ls_loop_handle (ob_dup_table_util.cpp:624) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=33] [DUP_TABLE_DIAG: Dup Interface Stat] tenant: 1002, ls: 1079, is_master: False, {dup_table_follower_read_succ_cnt_:780757, dup_table_follower_read_tablet_not_exist_cnt_:0, dup_table_follower_read_tablet_not_ready_cnt_:0, dup_table_follower_read_lease_expired_cnt_:71, dup_table_redo_sync_succ_cnt_:0, dup_table_redo_sync_fail_cnt_:0, dup_table_max_applying_scn_:{val:1723774806387358001, v:0}, dup_table_log_entry_cnt_:2761315, dup_table_log_entry_total_size_:1070044689, dup_table_log_replay_total_time_:0, dup_table_log_deser_total_time_:0, dup_table_lease_log_sync_total_time_:4042976, dup_table_tablet_log_sync_total_time_:2501518} [2024-08-16 10:20:07.049717] INFO [STORAGE.DUP_TABLE] print_lease_diag_info_log (ob_dup_table_lease.cpp:658) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=19] [DUP_TABLE_DIAG: Follower Lease Info] tenant: 1002, ls: 1079 , current_time = 2024-08-16 10:20:07.49702 [DUP_TABLE_DIAG: Follower Lease] is_expired=False, request_ts=1723774803747028, request_ts(date)=2024-08-16 10:20:03.747028, lease_expired_time=2024-08-16 10:20:09.747028, lease_interval_us=6000000, last_lease_scn={val:1723774804033739000, v:0}, lease_acquire_scn={val:1722500589629865001, v:0} [2024-08-16 10:20:07.050701] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=26][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:07.056431] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=28] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.056452] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:07.056476] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.056485] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.056500] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.057543] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=26] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:07.065459] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:07.065481] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:07.083874] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=35] PNIO [ratelimit] time: 1723774807083872, bytes: 1496437588309, bw: 0.168566 MB/s, add_ts: 1000425, add_bytes: 176829 [2024-08-16 10:20:07.084220] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8670-0-0] [lt=27] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:07.085402] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE6AE-0-0] [lt=35] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=10, avg_log_submit_to_flush_time=661, avg_log_submit_to_slide_time=1097) [2024-08-16 10:20:07.085425] INFO [PALF] read (log_cache.cpp:100) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=30] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:07.085480] INFO [LIB] stat (utility.h:1197) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=34] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=40, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:07.085503] INFO [LIB] stat (utility.h:1197) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=22] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=7, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:07.096043] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1ED-0-0] [lt=25][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509922, log_task={header:{begin_lsn:{lsn:18544231316}, end_lsn:{lsn:18544231438}, log_id:53509922, min_scn:{val:1723774806888489001, v:0}, max_scn:{val:1723774806888489001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544231194}, prev_proposal_id:7, committed_end_lsn:{lsn:18544231316}, data_checksum:1913811101, accum_checksum:2674175259, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774806995903, freeze_ts:1723774806995904, submit_ts:1723774806995908, flushed_ts:1723774806996526, gen_to_freeze cost time:1, gen_to_submit cost time:5, submit_to_flush cost time:618}, fs_cb_begin_ts=1723774807096030, log_life_time=100127) [2024-08-16 10:20:07.105199] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.105222] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:07.105238] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.105252] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.105276] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.110231] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=41] PNIO [ratelimit] time: 1723774807110229, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007591, add_bytes: 0 [2024-08-16 10:20:07.110512] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E2C-0-0] [lt=24] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11027, add_timestamp:1723774807110496}) [2024-08-16 10:20:07.110558] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676708-0-0] [lt=53] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11027, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:07.110700] INFO [SERVER] fill_tenant_ids_ (ob_all_virtual_tx_stat.cpp:139) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E2C-0-0] [lt=20] succeed to get tenant ids(all_tenants_=[1, 1001, 1002]) [2024-08-16 10:20:07.110962] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676708-0-0] [lt=25] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11027, locations=[{table_id:11027, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:07.111032] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676708-0-0] [lt=68] success to process renew task(task={tenant_id:1, table_id:11027, add_timestamp:1723774807110496}, locations=[{table_id:11027, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}, {table_id:11027, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774807110927, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:07.130103] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:07.130427] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=36] Wash time detail, (compute_wash_size_time=181, refresh_score_time=91, wash_time=24) [2024-08-16 10:20:07.130929] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2659][T1001_Occam][T1001][YB42C0A80080-00061CCCBF8D3866-0-0] [lt=5] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:07.133758] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Cache wash clean map node details(ret=0, clean_node_count=92, clean_time=0, clean_start_pos=503312, clean_num=62914) [2024-08-16 10:20:07.148356] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E2D-0-0] [lt=19][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=4, len=89, pos=62, bitmap=NULL) [2024-08-16 10:20:07.149430] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7E1-0-0] [lt=189][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:07.149507] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:07.149605] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=19, avg_log_batch_cnt=1, total_group_log_size=5766, avg_group_log_size=303) [2024-08-16 10:20:07.149638] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=5766) [2024-08-16 10:20:07.149653] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:07.149661] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:07.149668] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:07.149756] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=6] get_min_unreplayed_log_info(lsn={lsn:18544231438}, scn={val:1723774806888489002, v:0}, this={ls_id_:{id:1079}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774807096032, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:11904, ret_code_:0}}, next_to_submit_lsn_:{lsn:18544231438}, next_to_submit_scn_:{val:1723774806888489002, v:0}, base_lsn_:{lsn:13018324992}, base_scn_:{val:1720509584063969004, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd005000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:18544231316}, end_lsn:{lsn:18544231438}, read_buf:{buf_len_:2154496, buf_:0x7fa6cd005000}, block_size:67104768, log_storage_:0x7fa6fc6ef470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774806888489001, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774806888489001, v:0}, data_checksum:959419206, flag:0}}, init_mode_version:0, accumulate_checksum:2674175259, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774806888489001, v:0}}}}}) [2024-08-16 10:20:07.150159] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7E1-0-0] [lt=24] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:07.150537] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=5766) [2024-08-16 10:20:07.150777] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:120) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21] tenant info is invalid(ret=-4076, query_ts=0, standby_scn={val:0, v:0}, this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:20:07.150804] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=25][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:07.156508] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.156531] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:07.156541] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.156550] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.156564] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.158055] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B3D-0-0] [lt=81] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:07.158102] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B3D-0-0] [lt=44] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:07.158631] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=52] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:07.159172] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74467-0-0] [lt=10] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=11, avg_log_submit_to_flush_time=607, avg_log_submit_to_slide_time=1017) [2024-08-16 10:20:07.159198] INFO [PALF] read (log_cache.cpp:100) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:07.161447] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007F-000616D421E91A0F-0-0] [lt=22][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:07.164956] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF_DUMP](palf_id=1260, self="192.168.0.128:2882", [SlidingWindow]={palf_id:1260, self:"192.168.0.128:2882", lsn_allocator:{max_log_id:34644150, max_lsn:8828152760, max_scn:{val:1723774806929353002, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:8828152760}, reserved_buffer_size:41943040, available_buffer_size:33554432, readable_begin_lsn:{lsn:8786209720}}, last_submit_lsn:{lsn:8828152638}, last_submit_end_lsn:{lsn:8828152760}, last_submit_log_id:34644150, last_submit_log_pid:29, max_flushed_lsn:{lsn:8828152638}, max_flushed_end_lsn:{lsn:8828152760}, max_flushed_log_pid:29, committed_end_lsn:{lsn:8828152760}, last_slide_log_id:34644150, last_slide_scn:{val:1723774806929353002, v:0}, last_slide_lsn:{lsn:8828152638}, last_slide_end_lsn:{lsn:8828152760}, last_slide_log_pid:29, last_slide_log_accum_checksum:1522797858, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x7fa6bd554530}, [StateMgr]={this:0x7fa6bd5582b0, self:"192.168.0.128:2882", palf_id:1260, role:"LEADER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:29}, leader:"192.168.0.128:2882", leader_epoch:32, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:8469210191}, scan_disk_log_finished:true, last_check_start_id:34644151, is_changing_config_with_arb:false, reconfirm_start_time_us:1723617929285618, palf_role_change_cb:0x7fa6bd55eb70, allow_vote:true}, [ConfigMgr]={palf_id:1260, self:"192.168.0.128:2882", alive_paxos_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, alive_paxos_replica_num:3, log_ms_meta:{version:3, proposal_id:29, prev:{version:1, config:{config_version:{proposal_id:29, config_seq:443}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:29240, lock_time:1723646523720109}}, curr:{version:1, config:{config_version:{proposal_id:29, config_seq:444}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29240, lock_time:1723646524240102}}, prev_log_proposal_id:29, prev_lsn:{lsn:8514860172}, prev_mode_pid:29}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:29240, lock_type:0, added_list:0, removed_list:0, type:"UNLOCK_CONFIG_CHANGE"}, state:0, checking_barrier:{prev_log_proposal_id:29, prev_lsn:{lsn:8514860172}, prev_end_lsn:{lsn:8514860294}, prev_mode_pid:29}, reconfig_barrier:{prev_log_proposal_id:29, prev_lsn:{lsn:8514860172}, prev_end_lsn:{lsn:8514860294}, prev_mode_pid:29}, persistent_config_version:{proposal_id:29, config_seq:444}, ms_ack_list:0, resend_config_version:{proposal_id:29, config_seq:444}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1723617925582820, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1723617925582820, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:1722595738376965, this:0x7fa6bd555c70}, [ModeMgr]={palf_id:1260, self:"192.168.0.128:2882", applied_mode_meta:{version:1, proposal_id:29, mode_version:0, access_mode:1, ref_scn:{val:1720516541172103000, v:0}}, accepted_mode_meta:{version:1, proposal_id:29, mode_version:0, access_mode:1, ref_scn:{val:1720516541172103000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:29, mode_version:0, access_mode:1, ref_scn:{val:1720516541172103000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:1723617929485855, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1260, is_inited:true, min_block_max_scn:{val:1721971839646833003, v:0}, min_block_id:75, min_block_min_scn:{val:1721937225099170005, v:0}, base_lsn_for_block_gc:{lsn:8723619840}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:29}, log_config_meta:{version:3, proposal_id:29, prev:{version:1, config:{config_version:{proposal_id:29, config_seq:443}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:29240, lock_time:1723646523720109}}, curr:{version:1, config:{config_version:{proposal_id:29, config_seq:444}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29240, lock_time:1723646524240102}}, prev_log_proposal_id:29, prev_lsn:{lsn:8514860172}, prev_mode_pid:29}, log_snapshot_meta:{version:1, base_lsn:{lsn:8723619840}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:29, mode_version:0, access_mode:1, ref_scn:{val:1720516541172103000, v:0}}}, log_meta_storage:{log_tail:{lsn:1409024}, readable_log_tail:{lsn:1409024}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:8857829376}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1260}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1260/meta", dir_fd:192, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:65695744, block_header_serialize_buf_:0x7fa6bd55a438, flashback_version:0}, log_storage:{log_tail:{lsn:8828152760}, readable_log_tail:{lsn:8828152760}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:8790724608}, min_scn:{val:1723749999379189001, v:0}, curr_block_id:131, palf_id:1260}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1260/log", dir_fd:196, min_block_id:75, max_block_id:132, curr_writable_block_id:131}, logical_block_size_:67104768, curr_block_writable_size_:29676616, block_header_serialize_buf_:0x7fa6bd55c0b8, flashback_version:0}, palf_epoch:2, last_purge_throttling_ts:1723617929284852, this:0x7fa6bd5589f0}, [Reconfirm]={palf_id:1260, self:"192.168.0.128:2882", state:"FINISHED", new_proposal_id:29, prepare_log_ack_list:2"192.168.0.125:2882";"192.168.0.133:2882";, curr_paxos_follower_list:2{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, majority_cnt:2, majority_max_log_server:"192.168.0.128:2882", majority_max_accept_pid:28, majority_max_lsn:{lsn:8469424174}, saved_end_lsn:{lsn:8469424174}, last_submit_prepare_req_time_us:1723617929285921, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x7fa6bd558430}) [2024-08-16 10:20:07.165198] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=237] [PALF_DUMP](palf_id=1260, self="192.168.0.128:2882", ack_info_list=[{member:{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}, last_ack_time_us:1723774807150838, last_flushed_end_lsn:{lsn:8828152760}}, {member:{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}, last_ack_time_us:1723774807150597, last_flushed_end_lsn:{lsn:8828152760}}, {member:{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, last_ack_time_us:1723774807150752, last_flushed_end_lsn:{lsn:8828152760}}]) [2024-08-16 10:20:07.165218] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=16] current log_task status(palf_id=1260, self="192.168.0.128:2882", log_id=34644151, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[{member:{server:"192.168.0.125:2882", timestamp:1720516855942002, flag:0}, last_ack_time_us:1723774807150838, last_flushed_end_lsn:{lsn:8828152760}}, {member:{server:"192.168.0.128:2882", timestamp:1720517744170033, flag:0}, last_ack_time_us:1723774807150597, last_flushed_end_lsn:{lsn:8828152760}}, {member:{server:"192.168.0.133:2882", timestamp:1720518182712836, flag:0}, last_ack_time_us:1723774807150752, last_flushed_end_lsn:{lsn:8828152760}}]) [2024-08-16 10:20:07.184026] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=26] gc stale ls task succ [2024-08-16 10:20:07.198485] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E2E-0-0] [lt=24][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=1, len=32, pos=18, bitmap=NULL) [2024-08-16 10:20:07.205244] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.205269] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:20:07.205280] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.205295] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.205312] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.225169] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [2777][T1001_ObXAHbWor][T1001][Y0-0000000000000000-0-0] [lt=39] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-08-16 10:20:07.236102] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=16] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:07.236553] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=20] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:07.236600] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:07.236605] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=18] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:07.236948] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=23] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:07.236955] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=17] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:07.237114] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=26] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:07.240584] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=38] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=1) [2024-08-16 10:20:07.246773] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2752][T1001_TenantMet][T1001][Y0-0000000000000000-0-0] [lt=53] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=0, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:07.249584] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:07.249714] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=30] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:07.249730] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:07.249737] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:07.250869] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:07.256588] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.256610] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:07.256620] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.256631] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.256645] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.258548] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AB4-0-0] [lt=19] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:07.258602] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AB4-0-0] [lt=39] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774806764624003, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774806764624003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:07.284476] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8674-0-0] [lt=18] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:07.285087] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=592, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:07.285116] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:07.285139] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=32, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:07.285193] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=36, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:07.285575] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7446B-0-0] [lt=25] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:07.285592] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA7446B-0-0] [lt=16] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1120, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:07.296661] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1F7-0-0] [lt=7][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509924, log_task={header:{begin_lsn:{lsn:18544231560}, end_lsn:{lsn:18544231682}, log_id:53509924, min_scn:{val:1723774807158972000, v:0}, max_scn:{val:1723774807158972000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544231438}, prev_proposal_id:7, committed_end_lsn:{lsn:18544231560}, data_checksum:1913811101, accum_checksum:753801574, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774807196321, freeze_ts:1723774807196321, submit_ts:1723774807196326, flushed_ts:1723774807197026, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:700}, fs_cb_begin_ts=1723774807296647, log_life_time=100326) [2024-08-16 10:20:07.297342] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=39, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:20:07.300828] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=12] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:20:07.300858] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=26] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=5292, mem_used=39515136) [2024-08-16 10:20:07.305272] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.305294] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:07.305309] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.305319] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.305333] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.334300] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:07.334569] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=31] Wash time detail, (compute_wash_size_time=123, refresh_score_time=68, wash_time=16) [2024-08-16 10:20:07.337865] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=97, clean_time=0, clean_start_pos=566226, clean_num=62914) [2024-08-16 10:20:07.349656] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:07.349768] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:07.349788] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:07.349796] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:07.350947] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=25][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:07.356614] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.356637] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:07.356649] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.356658] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.356672] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.360155] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=8] get all snapshots(ret=0, snapshots.count()=192, snapshots=[{snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213191, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213192, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213193, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213194, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213195, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213196, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213197, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213198, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213199, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213200, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213201, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213202, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213203, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213204, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213205, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213206, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213207, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213208, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213209, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213210, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213211, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213212, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213213, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213214, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213215, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213216, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213217, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213218, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213219, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213220, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213221, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213222, comment:null}]) [2024-08-16 10:20:07.367188] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE86-0-0] [lt=19] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:07.367228] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE86-0-0] [lt=24] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774806870210004, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774806870210004, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:07.384830] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=20] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:07.396354] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF1FD-0-0] [lt=42] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1623, avg_group_log_size=147) [2024-08-16 10:20:07.405383] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.405410] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:20:07.405423] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.405436] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.405462] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.408128] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=21] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574648, offset=42149305}, end_cursor:ObLogCursor{file_id=65, log_id=18574649, offset=42149546}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516bc050, len:2631, log_data_len:241, seq:18574648, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:07.408834] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=31] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574649, offset=42149546}, end_cursor:ObLogCursor{file_id=65, log_id=18574650, offset=42149795}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751704050, len:2390, log_data_len:249, seq:18574649, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:07.410367] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA9-0-0] [lt=36] push gts task success(*task={:""}) [2024-08-16 10:20:07.418964] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCA9-0-0] [lt=19] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:07.422536] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=20] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574650, offset=42149795}, end_cursor:ObLogCursor{file_id=65, log_id=18574651, offset=42150044}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516e4050, len:2141, log_data_len:249, seq:18574650, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:07.423193] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=27] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574651, offset=42150044}, end_cursor:ObLogCursor{file_id=65, log_id=18574652, offset=42150293}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516f8050, len:1892, log_data_len:249, seq:18574651, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:07.426047] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=33] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:07.426105] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=26] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:07.430907] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=22] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1, need_refresh=false, gts_local_cache={srr:[mts=1723774806930265], gts:1723774806930093000, latest_srr:[mts=1723774807430876]}) [2024-08-16 10:20:07.430945] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=26] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_refresh=false, gts_local_cache={srr:[mts=1723774807413723], gts:1723774807412052001, latest_srr:[mts=1723774807430934]}) [2024-08-16 10:20:07.431175] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=12] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_refresh=false, gts_local_cache={srr:[mts=1723774806930532], gts:1723774806929353000, latest_srr:[mts=1723774807431147]}) [2024-08-16 10:20:07.440692] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=29] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=0) [2024-08-16 10:20:07.449740] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:07.449896] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=32] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:07.449919] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:07.449931] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:07.450616] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=673, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:07.450689] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=57] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=32, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:07.450713] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:07.451031] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=31][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:07.451124] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE6B9-0-0] [lt=30] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:07.451142] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE6B9-0-0] [lt=18] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1234, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:07.456688] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.456718] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=28] handle task succ(id={id:1260}) [2024-08-16 10:20:07.456733] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.456747] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.456766] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.482726] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [2517][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=31] run rewrite rule refresh task(rule_mgr_->tenant_id_=1002) [2024-08-16 10:20:07.491264] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7E1-0-0] [lt=45] [RS_MGR] force set rs list(master_rs="192.168.0.126:2882") [2024-08-16 10:20:07.491297] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7E1-0-0] [lt=24] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"192.168.0.128:2882", server_id:16, rs_addr:"192.168.0.126:2882", rs_server_status:1, epoch_id:11}, hb_response={zone:"zone2", server:"192.168.0.128:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1720514727760575, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=34) [2024-08-16 10:20:07.505466] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.505503] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=30] handle task succ(id={id:1260}) [2024-08-16 10:20:07.505520] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.505535] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.505556] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.505870] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8678-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:07.526746] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=21] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7034, mem_used=12478464) [2024-08-16 10:20:07.534174] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=32] start do ls ha handler(ls_id_array_=[{id:1261}, {id:1260}, {id:1079}, {id:1262}]) [2024-08-16 10:20:07.538430] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:07.538720] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash time detail, (compute_wash_size_time=136, refresh_score_time=88, wash_time=16) [2024-08-16 10:20:07.542025] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=24] Cache wash clean map node details(ret=0, clean_node_count=96, clean_time=0, clean_start_pos=629140, clean_num=62914) [2024-08-16 10:20:07.549813] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:07.549934] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:07.549957] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:07.549964] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:07.551106] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:07.556777] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.556806] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:20:07.556822] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.556835] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.556872] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=24] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.558362] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B45-0-0] [lt=32] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:07.563590] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31502][AutoLSLoc][T0][YB42C0A80080-00061CCC8EEBBF6D-0-0] [lt=87] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:07.565224] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF_DUMP](palf_id=1079, self="192.168.0.128:2882", [SlidingWindow]={palf_id:1079, self:"192.168.0.128:2882", lsn_allocator:{max_log_id:53509926, max_lsn:18544231926, max_scn:{val:1723774807389470000, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:18544231926}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:18502288886}}, last_submit_lsn:{lsn:18544231804}, last_submit_end_lsn:{lsn:18544231926}, last_submit_log_id:53509926, last_submit_log_pid:7, max_flushed_lsn:{lsn:18544231804}, max_flushed_end_lsn:{lsn:18544231926}, max_flushed_log_pid:7, committed_end_lsn:{lsn:18544231804}, last_slide_log_id:53509925, last_slide_scn:{val:1723774807158972001, v:0}, last_slide_lsn:{lsn:18544231682}, last_slide_end_lsn:{lsn:18544231804}, last_slide_log_pid:7, last_slide_log_accum_checksum:935224182, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x7fa6fc6e8530}, [StateMgr]={this:0x7fa6fc6ec2b0, self:"192.168.0.128:2882", palf_id:1079, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:7}, leader:"0.0.0.0:0", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:53509926, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x7fa6fc6f2b70, allow_vote:true}, [ConfigMgr]={palf_id:1079, self:"192.168.0.128:2882", alive_paxos_memberlist:3{server:"192.168.0.124:2882", timestamp:1, flag:0}{server:"192.168.0.127:2882", timestamp:1, flag:0}{server:"192.168.0.131:2882", timestamp:1, flag:0}, alive_paxos_replica_num:3, log_ms_meta:{version:3, proposal_id:7, prev:{version:1, config:{config_version:{proposal_id:7, config_seq:37}, log_sync_memberlist:3{server:"192.168.0.124:2882", timestamp:1, flag:0}{server:"192.168.0.127:2882", timestamp:1, flag:0}{server:"192.168.0.131:2882", timestamp:1, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:4, learner_array:[{server:"192.168.0.125:2882", timestamp:1720516537043883, flag:0}, {server:"192.168.0.126:2882", timestamp:1720513938601680, flag:0}, {server:"192.168.0.128:2882", timestamp:1720516609055334, flag:0}, {server:"192.168.0.133:2882", timestamp:1720516557506171, flag:0}]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:7, config_seq:38}, log_sync_memberlist:3{server:"192.168.0.124:2882", timestamp:1, flag:0}{server:"192.168.0.127:2882", timestamp:1, flag:0}{server:"192.168.0.131:2882", timestamp:1, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:3, learner_array:[{server:"192.168.0.125:2882", timestamp:1720516537043883, flag:0}, {server:"192.168.0.128:2882", timestamp:1720516609055334, flag:0}, {server:"192.168.0.133:2882", timestamp:1720516557506171, flag:0}]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:7, prev_lsn:{lsn:16645750544}, prev_mode_pid:7}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:7, config_seq:38}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1720516633177815, parent:"192.168.0.131:2882", parent_keepalive_time_us:1723774807030532, last_submit_register_req_time_us:-1, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x7fa6fc6e9c70}, [ModeMgr]={palf_id:1079, self:"192.168.0.128:2882", applied_mode_meta:{version:1, proposal_id:7, mode_version:0, access_mode:1, ref_scn:{val:1718350230743746000, v:0}}, accepted_mode_meta:{version:1, proposal_id:7, mode_version:0, access_mode:1, ref_scn:{val:1718350230743746000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:7, mode_version:0, access_mode:1, ref_scn:{val:1718350230743746000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1079, is_inited:true, min_block_max_scn:{val:1721964469082002002, v:0}, min_block_id:226, min_block_min_scn:{val:1721914383906628001, v:0}, base_lsn_for_block_gc:{lsn:18453811200}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:7}, log_config_meta:{version:3, proposal_id:7, prev:{version:1, config:{config_version:{proposal_id:7, config_seq:37}, log_sync_memberlist:3{server:"192.168.0.124:2882", timestamp:1, flag:0}{server:"192.168.0.127:2882", timestamp:1, flag:0}{server:"192.168.0.131:2882", timestamp:1, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:4, learner_array:[{server:"192.168.0.125:2882", timestamp:1720516537043883, flag:0}, {server:"192.168.0.126:2882", timestamp:1720513938601680, flag:0}, {server:"192.168.0.128:2882", timestamp:1720516609055334, flag:0}, {server:"192.168.0.133:2882", timestamp:1720516557506171, flag:0}]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:7, config_seq:38}, log_sync_memberlist:3{server:"192.168.0.124:2882", timestamp:1, flag:0}{server:"192.168.0.127:2882", timestamp:1, flag:0}{server:"192.168.0.131:2882", timestamp:1, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:3, learner_array:[{server:"192.168.0.125:2882", timestamp:1720516537043883, flag:0}, {server:"192.168.0.128:2882", timestamp:1720516609055334, flag:0}, {server:"192.168.0.133:2882", timestamp:1720516557506171, flag:0}]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:7, prev_lsn:{lsn:16645750544}, prev_mode_pid:7}, log_snapshot_meta:{version:1, base_lsn:{lsn:18453811200}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:7, mode_version:0, access_mode:1, ref_scn:{val:1718350230743746000, v:0}}}, log_meta_storage:{log_tail:{lsn:225280}, readable_log_tail:{lsn:225280}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:18588020736}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1079}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1079/meta", dir_fd:148, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:66879488, block_header_serialize_buf_:0x7fa6fc6ee438, flashback_version:0}, log_storage:{log_tail:{lsn:18544231926}, readable_log_tail:{lsn:18544231926}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:18520915968}, min_scn:{val:1723759521816553001, v:0}, curr_block_id:276, palf_id:1079}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1079/log", dir_fd:156, min_block_id:226, max_block_id:277, curr_writable_block_id:276}, logical_block_size_:67104768, curr_block_writable_size_:43788810, block_header_serialize_buf_:0x7fa6fc6f00b8, flashback_version:0}, palf_epoch:1, last_purge_throttling_ts:-1, this:0x7fa6fc6ec9f0}, [Reconfirm]={palf_id:1079, self:"192.168.0.128:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x7fa6fc6ec430}) [2024-08-16 10:20:07.565447] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=218][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1079, self="192.168.0.128:2882", tmp_server={server:"192.168.0.124:2882", timestamp:1, flag:0}) [2024-08-16 10:20:07.565460] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF_DUMP](palf_id=1079, self="192.168.0.128:2882", ack_info_list=[]) [2024-08-16 10:20:07.565468] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1079, self="192.168.0.128:2882", tmp_server={server:"192.168.0.124:2882", timestamp:1, flag:0}) [2024-08-16 10:20:07.565474] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=5] current log_task status(palf_id=1079, self="192.168.0.128:2882", log_id=53509926, log_task={header:{begin_lsn:{lsn:18544231804}, end_lsn:{lsn:18544231926}, log_id:53509926, min_scn:{val:1723774807389470000, v:0}, max_scn:{val:1723774807389470000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544231682}, prev_proposal_id:7, committed_end_lsn:{lsn:18544231804}, data_checksum:1913811101, accum_checksum:975831299, is_padding_log:false, is_raw_write:false}, state_map:{val:26}, ref_cnt:0, gen_ts:1723774807396323, freeze_ts:1723774807396323, submit_ts:1723774807396334, flushed_ts:1723774807397143, gen_to_freeze cost time:0, gen_to_submit cost time:11, submit_to_flush cost time:809}, ack_info_list=[]) [2024-08-16 10:20:07.565511] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF_DUMP](palf_id=1261, self="192.168.0.128:2882", [SlidingWindow]={palf_id:1261, self:"192.168.0.128:2882", lsn_allocator:{max_log_id:33675640, max_lsn:32601572058, max_scn:{val:1723774807405598000, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:32601572058}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:32559629018}}, last_submit_lsn:{lsn:32601571936}, last_submit_end_lsn:{lsn:32601572058}, last_submit_log_id:33675640, last_submit_log_pid:5, max_flushed_lsn:{lsn:32601571936}, max_flushed_end_lsn:{lsn:32601572058}, max_flushed_log_pid:5, committed_end_lsn:{lsn:32601572058}, last_slide_log_id:33675640, last_slide_scn:{val:1723774807405598000, v:0}, last_slide_lsn:{lsn:32601571936}, last_slide_end_lsn:{lsn:32601572058}, last_slide_log_pid:5, last_slide_log_accum_checksum:1012244929, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x7fa6fe292530}, [StateMgr]={this:0x7fa6fe2962b0, self:"192.168.0.128:2882", palf_id:1261, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:5}, leader:"192.168.0.125:2882", leader_epoch:9, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:32276646160}, scan_disk_log_finished:true, last_check_start_id:33675641, is_changing_config_with_arb:false, reconfirm_start_time_us:1722594182152738, palf_role_change_cb:0x7fa6fe29cb70, allow_vote:true}, [ConfigMgr]={palf_id:1261, self:"192.168.0.128:2882", alive_paxos_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, alive_paxos_replica_num:3, log_ms_meta:{version:3, proposal_id:5, prev:{version:1, config:{config_version:{proposal_id:4, config_seq:415}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29145, lock_time:1723617733782611}}, curr:{version:1, config:{config_version:{proposal_id:5, config_seq:416}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29145, lock_time:1723617733782611}}, prev_log_proposal_id:4, prev_lsn:{lsn:32276646038}, prev_mode_pid:5}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:4, prev_lsn:{lsn:32276415355}, prev_end_lsn:{lsn:32276415514}, prev_mode_pid:4}, reconfig_barrier:{prev_log_proposal_id:4, prev_lsn:{lsn:32276415355}, prev_end_lsn:{lsn:32276415514}, prev_mode_pid:4}, persistent_config_version:{proposal_id:5, config_seq:416}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1723774806264349, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1723774806264349, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:1722931663742194, this:0x7fa6fe293c70}, [ModeMgr]={palf_id:1261, self:"192.168.0.128:2882", applied_mode_meta:{version:1, proposal_id:5, mode_version:0, access_mode:1, ref_scn:{val:1720516541190358000, v:0}}, accepted_mode_meta:{version:1, proposal_id:5, mode_version:0, access_mode:1, ref_scn:{val:1720516541190358000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:5, mode_version:0, access_mode:1, ref_scn:{val:1720516541190358000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1261, is_inited:true, min_block_max_scn:{val:1721957059006967058, v:0}, min_block_id:421, min_block_min_scn:{val:1721956977387008021, v:0}, base_lsn_for_block_gc:{lsn:32545812480}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:5}, log_config_meta:{version:3, proposal_id:5, prev:{version:1, config:{config_version:{proposal_id:4, config_seq:415}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29145, lock_time:1723617733782611}}, curr:{version:1, config:{config_version:{proposal_id:5, config_seq:416}, log_sync_memberlist:3{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, log_sync_replica_num:3, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:29145, lock_time:1723617733782611}}, prev_log_proposal_id:4, prev_lsn:{lsn:32276646038}, prev_mode_pid:5}, log_snapshot_meta:{version:1, base_lsn:{lsn:32545812480}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:5, mode_version:0, access_mode:1, ref_scn:{val:1720516541190358000, v:0}}}, log_meta_storage:{log_tail:{lsn:1191936}, readable_log_tail:{lsn:1191936}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:32612917248}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1261}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1261/meta", dir_fd:199, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:65912832, block_header_serialize_buf_:0x7fa6fe298438, flashback_version:0}, log_storage:{log_tail:{lsn:32601572058}, readable_log_tail:{lsn:32601572058}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:32545812480}, min_scn:{val:1723732336322565001, v:0}, curr_block_id:485, palf_id:1261}, block_mgr:{log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1002/1261/log", dir_fd:200, min_block_id:421, max_block_id:486, curr_writable_block_id:485}, logical_block_size_:67104768, curr_block_writable_size_:11345190, block_header_serialize_buf_:0x7fa6fe29a0b8, flashback_version:0}, palf_epoch:3, last_purge_throttling_ts:1723452062503097, this:0x7fa6fe2969f0}, [Reconfirm]={palf_id:1261, self:"192.168.0.128:2882", state:"FINISHED", new_proposal_id:4, prepare_log_ack_list:2"192.168.0.125:2882";"192.168.0.133:2882";, curr_paxos_follower_list:2{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, majority_cnt:2, majority_max_log_server:"192.168.0.128:2882", majority_max_accept_pid:3, majority_max_lsn:{lsn:30467531174}, saved_end_lsn:{lsn:30467531174}, last_submit_prepare_req_time_us:1722594182153177, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x7fa6fe296430}) [2024-08-16 10:20:07.565682] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=167] [PALF_DUMP](palf_id=1261, self="192.168.0.128:2882", ack_info_list=[{member:{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}, {member:{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}, {member:{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}]) [2024-08-16 10:20:07.565698] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=15] current log_task status(palf_id=1261, self="192.168.0.128:2882", log_id=33675641, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[{member:{server:"192.168.0.125:2882", timestamp:1720516876519282, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}, {member:{server:"192.168.0.128:2882", timestamp:1720517744185691, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}, {member:{server:"192.168.0.133:2882", timestamp:1720518182735295, flag:0}, last_ack_time_us:1723617909269533, last_flushed_end_lsn:{lsn:32276646160}}]) [2024-08-16 10:20:07.568589] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [2672][T1001_PalfGC][T1001][Y0-0000000000000000-0-0] [lt=56] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):2150, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):2150, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}) [2024-08-16 10:20:07.578931] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 293.00) [2024-08-16 10:20:07.578955] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=18] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:07.579032] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=42] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.99, rt: 511.25) [2024-08-16 10:20:07.579047] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=13] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:07.579090] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579100] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579107] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579124] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579133] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579140] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579147] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579151] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579155] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579159] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579163] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579168] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579172] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579179] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579188] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.579195] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:07.584948] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=7] ====== check clog disk timer task ====== [2024-08-16 10:20:07.584974] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=22] get_disk_usage(ret=0, capacity(MB):=2150, used(MB):=0) [2024-08-16 10:20:07.584993] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=12] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=676457349, need_update_checkpoint_scn=false) [2024-08-16 10:20:07.586052] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=28] ====== check clog disk timer task ====== [2024-08-16 10:20:07.586091] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=36] get_disk_usage(ret=0, capacity(MB):=0, used(MB):=0) [2024-08-16 10:20:07.586107] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=11] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=0, need_update_checkpoint_scn=false) [2024-08-16 10:20:07.591618] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=26][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:07.593996] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=38] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:07.594098] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=22][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:07.594125] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=25][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:07.594137] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=11] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:07.600603] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF203-0-0] [lt=24] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:07.600644] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF203-0-0] [lt=20] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=10, avg_log_submit_to_flush_time=733, avg_log_submit_to_slide_time=110527) [2024-08-16 10:20:07.600661] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF203-0-0] [lt=15][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509926, log_task={header:{begin_lsn:{lsn:18544231804}, end_lsn:{lsn:18544231926}, log_id:53509926, min_scn:{val:1723774807389470000, v:0}, max_scn:{val:1723774807389470000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544231682}, prev_proposal_id:7, committed_end_lsn:{lsn:18544231804}, data_checksum:1913811101, accum_checksum:975831299, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774807396323, freeze_ts:1723774807396323, submit_ts:1723774807396334, flushed_ts:1723774807397143, gen_to_freeze cost time:0, gen_to_submit cost time:11, submit_to_flush cost time:809}, fs_cb_begin_ts=1723774807600629, log_life_time=204306) [2024-08-16 10:20:07.600691] INFO [PALF] read (log_cache.cpp:100) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:07.601364] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=19, stat_interval=1000000, avg (count/cost)=32, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:07.605560] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.605582] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:07.605595] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.605604] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.605620] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.607794] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=24] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:07.617886] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] ====== checkpoint timer task ====== [2024-08-16 10:20:07.618349] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=34] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60741, mem_used=137263104) [2024-08-16 10:20:07.618838] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=18] succ to get rec scn(*this={this:0x7fa707804030, ls_id:{id:1079}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1722510647991558001, v:0}, last_push_gc_task_ts_:1722511534898926, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:103}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:07.618871] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=32] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596c080, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1722511534901545, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596c080, snapshot_version:{val:1722511534901545, v:0}, ls_id:{id:1079}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:07.618910] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=31] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1722497541268834001, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1722497541268834001, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1079}) [2024-08-16 10:20:07.618970] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:265) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=21] [FREEZE]get rec_scn from MdsTableMgr(ret=0, ret="OB_SUCCESS", min_rec_scn={val:1723723445809886004, v:0}, order_flusher_for_one={high_priority_flusher:size:1, ordered_tablets:{{tablet_id:{id:208204}, rec_scn:{val:1723723445809886004, v:0}}, }, *this={this:0x7fa6fa246510, is_inited:true, freezing_scn:{val:1721997282211836001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774804660672003, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:07.619047] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=73] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="MDS_TABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={this:0x7fa6fa246510, is_inited:true, freezing_scn:{val:1721997282211836001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774804660672003, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}, min_rec_scn={val:1723723445809886004, v:0}, ls_id_={id:1079}) [2024-08-16 10:20:07.619139] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=68] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723723202639465000, v:0}, checkpoint_scn_in_ls_meta={val:1723723202639465000, v:0}, ls_id={id:1079}, service_type="DUP_TABLE") [2024-08-16 10:20:07.619167] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=14][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:18453811200}, new_base_lsn={lsn:18453811200}, lsn={lsn:18453811200}) [2024-08-16 10:20:07.619181] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=13] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:18453811200}, ls->get_ls_id()={id:1079}) [2024-08-16 10:20:07.620041] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] succ to get rec scn(*this={this:0x7fa7078b6030, ls_id:{id:1260}, tenant_id:1002, state:"L_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1723690101339443000, v:0}, last_push_gc_task_ts_:1723691902664029, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:6460}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:07.620065] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=23] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596eab0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1723691902666264, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596eab0, snapshot_version:{val:1723691902666264, v:0}, ls_id:{id:1260}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:07.620099] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=30] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1723690100831412000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1723690100831412000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1260}) [2024-08-16 10:20:07.621642] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:265) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=14] [FREEZE]get rec_scn from MdsTableMgr(ret=0, ret="OB_SUCCESS", min_rec_scn={val:1723723302639709002, v:0}, order_flusher_for_one={high_priority_flusher:size:1, ordered_tablets:{{tablet_id:{id:1152921504606868225}, rec_scn:{val:1723723302639709002, v:0}}, }, *this={this:0x7fa6bd53e510, is_inited:true, freezing_scn:{val:1723690100831412000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1260}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202608224000, v:0}, clog_base_lsn:{lsn:8723619840}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774804660672003, v:0}, tablet_change_checkpoint_scn:{val:1723719123293042000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723639313794159001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1260, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:07.621698] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=53] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x7fa6bd53e1b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x7fa6bd543cd0, table_type:1, memtable_head:149, memtable_tail:150, t3m:0x7fa7118b2030, tables:[null, null, null, null, null, 0x7fa70dc06cc0, null, null, null, null, null, null, null, null, null, null]}, is_freezing:false, ls_id:{id:1260}, mini_merge_recycle_commit_versions_ts:1723723202700999, tx_data_table:0x7fa6bd5451d0, ls_tablet_svr:0x7fa6bd53e190, slice_allocator:0x7fa6bd545210}, min_rec_scn={val:1723723202608224000, v:0}, ls_id_={id:1260}) [2024-08-16 10:20:07.626941] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=27] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723723202608224000, v:0}, checkpoint_scn_in_ls_meta={val:1723723202608224000, v:0}, ls_id={id:1260}, service_type="TRANS_SERVICE") [2024-08-16 10:20:07.626981] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:8723619840}, new_base_lsn={lsn:8723619840}, lsn={lsn:8723619840}) [2024-08-16 10:20:07.627008] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=25] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:8723619840}, ls->get_ls_id()={id:1260}) [2024-08-16 10:20:07.627932] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] succ to get rec scn(*this={this:0x7fa70da04030, ls_id:{id:1261}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1723712452247151000, v:0}, last_push_gc_task_ts_:1723712455858054, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:6477}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:07.627961] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=27] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596ce90, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1723723202513573, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596ce90, snapshot_version:{val:1723723202513573, v:0}, ls_id:{id:1261}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:07.627981] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=16] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1723654533322558000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1723654533322558000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1261}) [2024-08-16 10:20:07.628483] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=9] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x7fa6fe204510, is_inited:true, freezing_scn:{val:1723752041922970000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1261}, ls_create_status:1, clog_checkpoint_scn:{val:1723752188579788000, v:0}, clog_base_lsn:{lsn:32545812480}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774804660672003, v:0}, tablet_change_checkpoint_scn:{val:1723719128263216001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723617733349446001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:4, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1261}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1261, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:07.628534] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=48] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x7fa6fe2041b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x7fa6fe209cd0, table_type:1, memtable_head:173, memtable_tail:174, t3m:0x7fa7118b2030, tables:[null, null, null, null, null, null, null, null, null, null, null, null, null, 0x7fa70dc09900, null, null]}, is_freezing:false, ls_id:{id:1261}, mini_merge_recycle_commit_versions_ts:1723752042418277, tx_data_table:0x7fa6fe20b1d0, ls_tablet_svr:0x7fa6fe204190, slice_allocator:0x7fa6fe20b210}, min_rec_scn={val:1723752188579788000, v:0}, ls_id_={id:1261}) [2024-08-16 10:20:07.633800] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=26] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723752188579788000, v:0}, checkpoint_scn_in_ls_meta={val:1723752188579788000, v:0}, ls_id={id:1261}, service_type="TRANS_SERVICE") [2024-08-16 10:20:07.633827] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:32545812480}, new_base_lsn={lsn:32545812480}, lsn={lsn:32545812480}) [2024-08-16 10:20:07.633837] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:32545812480}, ls->get_ls_id()={id:1261}) [2024-08-16 10:20:07.634731] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] succ to get rec scn(*this={this:0x7fa70dab6030, ls_id:{id:1262}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1723712432019955000, v:0}, last_push_gc_task_ts_:1723712435838837, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:6328}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:07.634756] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=24] tx ctx memtable get rec scn(this={ObITable:{this:0x7fa71596dca0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1723712435841010, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x7fa71596dca0, snapshot_version:{val:1723712435841010, v:0}, ls_id:{id:1262}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-08-16 10:20:07.634774] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=14] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1723657339856816000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1723657339856816000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1262}) [2024-08-16 10:20:07.636314] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:265) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=9] [FREEZE]get rec_scn from MdsTableMgr(ret=0, ret="OB_SUCCESS", min_rec_scn={val:1723723207973113002, v:0}, order_flusher_for_one={high_priority_flusher:size:1, ordered_tablets:{{tablet_id:{id:209134}, rec_scn:{val:1723723207973113002, v:0}}, }, *this={this:0x7fa6be4fc510, is_inited:true, freezing_scn:{val:1723656976361279000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774804660672003, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:07.636393] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=75] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="MDS_TABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={this:0x7fa6be4fc510, is_inited:true, freezing_scn:{val:1723656976361279000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774804660672003, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}, min_rec_scn={val:1723723207973113002, v:0}, ls_id_={id:1262}) [2024-08-16 10:20:07.641543] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=41] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1723723207973113002, v:0}, checkpoint_scn_in_ls_meta={val:1723723207973113002, v:0}, ls_id={id:1262}, service_type="TRANS_SERVICE") [2024-08-16 10:20:07.641571] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, curr_base_lsn={lsn:8253886464}, new_base_lsn={lsn:8253886464}, lsn={lsn:8253886464}) [2024-08-16 10:20:07.641586] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=14] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:8253886464}, ls->get_ls_id()={id:1262}) [2024-08-16 10:20:07.641597] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:191) [2558][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=9] succeed to update_clog_checkpoint(ret=0, ls_cnt=4) [2024-08-16 10:20:07.642349] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=22] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=2736759, replace_num=31457) [2024-08-16 10:20:07.642374] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=22] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=10) [2024-08-16 10:20:07.645295] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [31779][T1_ArcSender][T1][YB42C0A80080-00061CCC8D41D0C5-0-0] [lt=20][errcode=-4018] try free send task failed(ret=-4018) [2024-08-16 10:20:07.649901] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:07.650022] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:07.650038] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:07.650045] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:07.651193] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:07.651220] INFO [LIB] stat (utility.h:1197) [19129][T1002_ApplySrv2][T1002][YB42C0A80080-00061CCCC52E0953-0-0] [lt=6] [APPLY STAT CB APPEND COST TIME](cur_stat_count=52, stat_interval=5000000, avg (count/cost)=93, this=0x7fa70bda0280, extra_info="") [2024-08-16 10:20:07.651238] INFO [LIB] stat (utility.h:1197) [19129][T1002_ApplySrv2][T1002][YB42C0A80080-00061CCCC52E0953-0-0] [lt=17] [APPLY STAT CB IN QUEUE TIME](cur_stat_count=52, stat_interval=5000000, avg (count/cost)=1279, this=0x7fa70bda04b0, extra_info="") [2024-08-16 10:20:07.651246] INFO [LIB] stat (utility.h:1197) [19129][T1002_ApplySrv2][T1002][YB42C0A80080-00061CCCC52E0953-0-0] [lt=7] [APPLY STAT CB WAIT COMMIT TIME](cur_stat_count=52, stat_interval=5000000, avg (count/cost)=0, this=0x7fa70bda06e0, extra_info="") [2024-08-16 10:20:07.651251] INFO [LIB] stat (utility.h:1197) [19129][T1002_ApplySrv2][T1002][YB42C0A80080-00061CCCC52E0953-0-0] [lt=5] [APPLY STAT CB EXECUTE TIME](cur_stat_count=52, stat_interval=5000000, avg (count/cost)=-503, this=0x7fa70bda0910, extra_info="") [2024-08-16 10:20:07.651256] INFO [LIB] stat (utility.h:1197) [19129][T1002_ApplySrv2][T1002][YB42C0A80080-00061CCCC52E0953-0-0] [lt=4] [APPLY STAT CB TOTAL TIME](cur_stat_count=52, stat_interval=5000000, avg (count/cost)=869, this=0x7fa70bda0b40, extra_info="") [2024-08-16 10:20:07.656849] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.656869] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:07.656879] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.656888] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.656902] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.658957] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=510, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:07.659025] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=15] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=35, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:07.659403] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74475-0-0] [lt=37] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:07.659425] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74475-0-0] [lt=21] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=985, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:07.660703] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [2790][T1001_ArcSender][T1001][YB42C0A80080-00061CCC9351D0C5-0-0] [lt=20][errcode=-4018] try free send task failed(ret=-4018) [2024-08-16 10:20:07.677195] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=24] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:07.700817] INFO [STORAGE.DUP_TABLE] follower_try_acquire_lease (ob_dup_table_lease.cpp:500) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=9] [DupLeaseLife] The follower can get new lease from this lease log(ret=0, ls_id={id:1079}, lease_log_scn={val:1723774807482778001, v:0}, self_addr="192.168.0.128:2882", acquire_new_lease=false, follower_lease_info_={durable_lease_:{request_ts_:1723774807349910, lease_interval_us_:6000000}, lease_expired_ts_:1723774813349910, last_lease_scn_:{val:1723774807482778001, v:0}, lease_acquire_scn_:{val:1722500589629865001, v:0}}) [2024-08-16 10:20:07.705637] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.705667] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:20:07.705681] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.705719] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=36] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.705736] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.742523] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:07.742818] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=57] Wash time detail, (compute_wash_size_time=162, refresh_score_time=73, wash_time=18) [2024-08-16 10:20:07.746186] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Cache wash clean map node details(ret=0, clean_node_count=96, clean_time=0, clean_start_pos=692054, clean_num=62914) [2024-08-16 10:20:07.749965] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] try gc retain ctx [2024-08-16 10:20:07.750000] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:07.750118] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=46, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:07.750152] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:07.750160] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:07.750167] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:07.750172] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:07.751261] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F7368F-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=6, avg_log_gen_to_submit_time=65, avg_log_submit_to_flush_time=751, avg_log_submit_to_slide_time=1183) [2024-08-16 10:20:07.751275] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=35][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:07.756921] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=25] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.756942] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:07.756952] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.756963] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.756976] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.771004] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=20] ====== tenant freeze timer task ====== [2024-08-16 10:20:07.771034] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=18][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:07.800949] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF20F-0-0] [lt=29] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:07.801011] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF20F-0-0] [lt=33] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:07.801019] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF20F-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=100530, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:07.801026] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF20F-0-0] [lt=4][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509929, log_task={header:{begin_lsn:{lsn:18544232451}, end_lsn:{lsn:18544232573}, log_id:53509929, min_scn:{val:1723774807482778002, v:0}, max_scn:{val:1723774807482778002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544232048}, prev_proposal_id:7, committed_end_lsn:{lsn:18544232451}, data_checksum:1913811101, accum_checksum:2074241866, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774807700696, freeze_ts:1723774807700696, submit_ts:1723774807700703, flushed_ts:1723774807701330, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:627}, fs_cb_begin_ts=1723774807801000, log_life_time=100304) [2024-08-16 10:20:07.801668] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1501) [2024-08-16 10:20:07.805719] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.805741] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:07.805751] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.805761] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.805774] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.842461] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=9) [2024-08-16 10:20:07.850046] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:07.850172] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=41] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:07.850207] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=32] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:07.850216] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:07.850847] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=15] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:07.851371] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:07.857022] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.857044] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:07.857055] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.857065] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.857096] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=22] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.862174] INFO [SERVER] try_reload_schema (ob_server_schema_updater.cpp:440) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=10] schedule fetch new schema task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:07.862196] INFO [SERVER] do_heartbeat_event (ob_heartbeat.cpp:268) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=21] try reload schema success(schema_version=1722933080671200, refresh_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_ret=0) [2024-08-16 10:20:07.862206] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:255) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E81-0-0] [lt=16] [REFRESH_SCHEMA] start to process schema refresh task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:07.862230] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:280) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E81-0-0] [lt=20] [REFRESH_SCHEMA] local schema info is newer, no need to refresh schema(ret=0, ret="OB_SUCCESS", local_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:07.873837] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [31770][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=62] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1, server_version={version:{val:1723774802931273000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0}, server_version_delta=4942559, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.126:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29977711, succ_cluster_heartbeat_count=29977693, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774342455102000, v:0}, local_cluster_delta=465418730, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:20:07.878008] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [2536][T1002_LockWaitM][T1002][Y0-0000000000000000-0-0] [lt=34] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-08-16 10:20:07.901617] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=39] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=614, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:07.905804] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.905827] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:07.905847] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.905857] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.905872] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.909263] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2520][T1002_TenantMet][T1002][Y0-0000000000000000-0-0] [lt=44] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:17374, free_obj_hold_cnt:593, allocator used:71436792, allocator total:73453184}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:1994, free_obj_hold_cnt:6, allocator used:131182000, allocator total:131182000}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():63488, full allocator used:1600, full allocator total:129024}, tablets_mem=204764208, tablets_mem_limit=1288490180, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:7, allocator used:17024, allocator total:65408}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:106, free_obj_hold_cnt:5057, allocator used:10243392, allocator total:10596096}, pending_cnt=0, wait_gc_count=0, tablet count=19368) [2024-08-16 10:20:07.909488] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [31504][DumpLSLoc][T0][YB42C0A80080-00061CCC8F16CA29-0-0] [lt=316] [LS_LOCATION]dump tenant ls location caches(tenant_id=1, tenant_ls_locations=[{cache_key:{tenant_id:1, ls_id:{id:1}, cluster_id:1713182723}, renew_time:1723774807563674, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:11}, {server:"192.168.0.129:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.134:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}]) [2024-08-16 10:20:07.909535] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [31504][DumpLSLoc][T0][YB42C0A80080-00061CCC8F16CA29-0-0] [lt=44] [LS_LOCATION]dump tenant ls location caches(tenant_id=1001, tenant_ls_locations=[{cache_key:{tenant_id:1001, ls_id:{id:1}, cluster_id:1713182723}, renew_time:1723774807566613, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:7}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}]) [2024-08-16 10:20:07.909636] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [31504][DumpLSLoc][T0][YB42C0A80080-00061CCC8F16CA29-0-0] [lt=19] [LS_LOCATION]dump tenant ls location caches(tenant_id=1002, tenant_ls_locations=[{cache_key:{tenant_id:1002, ls_id:{id:1312}, cluster_id:1713182723}, renew_time:1723105980208625, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1294}, cluster_id:1713182723}, renew_time:1722595652102251, replica_locations:[{server:"192.168.0.124:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1363}, cluster_id:1713182723}, renew_time:1723646553055685, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1292}, cluster_id:1713182723}, renew_time:1722594499160155, replica_locations:[{server:"192.168.0.124:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1286}, cluster_id:1713182723}, renew_time:1722594010522003, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1309}, cluster_id:1713182723}, renew_time:1722832077430884, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1300}, cluster_id:1713182723}, renew_time:1722595777348102, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1325}, cluster_id:1713182723}, renew_time:1723452113048028, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1350}, cluster_id:1713182723}, renew_time:1723617735379749, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1316}, cluster_id:1713182723}, renew_time:1723451932706270, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1280}, cluster_id:1713182723}, renew_time:1722593548703641, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:7}]}, {cache_key:{tenant_id:1002, ls_id:{id:1291}, cluster_id:1713182723}, renew_time:1722594313540279, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1290}, cluster_id:1713182723}, renew_time:1722594318555608, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.134:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1339}, cluster_id:1713182723}, renew_time:1723516963339645, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1284}, cluster_id:1713182723}, renew_time:1722593985405302, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.134:2882", role:2, sql_port:2881, replica_type:16, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1344}, cluster_id:1713182723}, renew_time:1723517113628829, replica_locations:[{server:"192.168.0.125:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1359}, cluster_id:1713182723}, renew_time:1723618040940888, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1330}, cluster_id:1713182723}, renew_time:1723516813046198, replica_locations:[{server:"192.168.0.124:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1261}, cluster_id:1713182723}, renew_time:1723774807570321, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:5}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1028}, cluster_id:1713182723}, renew_time:1723774807570314, replica_locations:[{server:"192.168.0.124:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:6}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1361}, cluster_id:1713182723}, renew_time:1723639351369428, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1271}, cluster_id:1713182723}, renew_time:1720533509017285, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1305}, cluster_id:1713182723}, renew_time:1722595958684187, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1269}, cluster_id:1713182723}, renew_time:1720533468951712, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1348}, cluster_id:1713182723}, renew_time:1723617720354819, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1337}, cluster_id:1713182723}, renew_time:1723516903210675, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1298}, cluster_id:1713182723}, renew_time:1722595737258877, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1329}, cluster_id:1713182723}, renew_time:1723516803029599, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1314}, cluster_id:1713182723}, renew_time:1723451932706267, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:3}, {server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1268}, cluster_id:1713182723}, renew_time:1720533488989721, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1274}, cluster_id:1713182723}, renew_time:1720591144595730, replica_locations:[{server:"192.168.0.127:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1002, ls_id:{id:1331}, cluster_id:1713182723}, renew_time:1723516803029649, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"192.168.0.128:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"192.168.0.133:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}]) [2024-08-16 10:20:07.909851] INFO [SHARE] blacklist_loop_ (ob_server_blacklist.cpp:313) [31526][Blacklist][T0][Y0-0000000000000000-0-0] [lt=35] blacklist_loop exec finished(cost_time=19, is_enabled=true, send_cnt=0) [2024-08-16 10:20:07.910007] INFO [CLOG] do_thread_task_ (ob_remote_fetch_log_worker.cpp:250) [2762][T1001_RFLWorker][T1001][YB42C0A80080-00061CCC92E1D0C5-0-0] [lt=18] ObRemoteFetchWorker is running(thread_index=0) [2024-08-16 10:20:07.931523] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=16] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:07.946672] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=31] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:07.946945] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=37] Wash time detail, (compute_wash_size_time=133, refresh_score_time=74, wash_time=16) [2024-08-16 10:20:07.950126] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:07.950200] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=23] Cache wash clean map node details(ret=0, clean_node_count=89, clean_time=0, clean_start_pos=754968, clean_num=62914) [2024-08-16 10:20:07.950249] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:07.950266] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:07.950277] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:07.951444] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:07.957056] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:07.957100] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=42] handle task succ(id={id:1260}) [2024-08-16 10:20:07.957112] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:07.957125] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:07.957146] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:07.962225] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=16] ====== check clog disk timer task ====== [2024-08-16 10:20:07.962253] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=20] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:07.962276] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=9] cannot_recycle_log_size statistics(cannot_recycle_log_size=391682457, threshold=6088116142, need_update_checkpoint_scn=false) [2024-08-16 10:20:07.971116] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007C-000616D3CE4B970C-0-0] [lt=20][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:07.997866] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=4] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:07.997919] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=42] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:07.998062] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:07.998106] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=36][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:07.998120] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:07.998133] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:08.001038] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF21B-0-0] [lt=34][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509931, log_task={header:{begin_lsn:{lsn:18544232695}, end_lsn:{lsn:18544232817}, log_id:53509931, min_scn:{val:1723774807889740000, v:0}, max_scn:{val:1723774807889740000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544232573}, prev_proposal_id:7, committed_end_lsn:{lsn:18544232695}, data_checksum:1913811101, accum_checksum:2194660138, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774807900948, freeze_ts:1723774807900948, submit_ts:1723774807900958, flushed_ts:1723774807901689, gen_to_freeze cost time:0, gen_to_submit cost time:10, submit_to_flush cost time:731}, fs_cb_begin_ts=1723774808001026, log_life_time=100078) [2024-08-16 10:20:08.005875] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.005896] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:08.005907] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.005922] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.005939] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.012858] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=25] gc stale ls task succ [2024-08-16 10:20:08.012909] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=4] gc stale ls task succ [2024-08-16 10:20:08.012929] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=11] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:08.012948] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=13] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:08.012958] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=7] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:08.017139] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:08.017164] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:08.031332] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C2C-0-0] [lt=60] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774808031287) [2024-08-16 10:20:08.042565] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=30] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=8) [2024-08-16 10:20:08.050199] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:08.050330] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:08.050347] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:08.050355] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:08.051531] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:08.056434] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80080-00061CCCC6B436C3-0-0] [lt=18][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:08.057160] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.057189] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:20:08.057205] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.057220] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.057237] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.057879] INFO [STORAGE.TRANS] print_stat_ (ob_black_list.cpp:398) [31842][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=22] start to print blacklist info [2024-08-16 10:20:08.058052] INFO [STORAGE.TRANS] run1 (ob_black_list.cpp:194) [31842][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=18] ls blacklist refresh finish(cost_time=12735) [2024-08-16 10:20:08.058374] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=27] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:08.059672] INFO [CLOG] handle_submit_task_ (ob_log_replay_service.cpp:1208) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=16] succ to submit log task to replay service(to_submit_lsn={lsn:8394853520}, to_submit_scn={val:1723774807766423002, v:0}, replay_status={ls_id_:{id:1262}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774808059632, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10679383, ret_code_:0}}, next_to_submit_lsn_:{lsn:8394853586}, next_to_submit_scn_:{val:1723774807766423003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516551920929002, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69ec05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8394853464}, end_lsn:{lsn:8394853586}, read_buf:{buf_len_:3682304, buf_:0x7fa69ec05000}, block_size:67104768, log_storage_:0x7fa6be533470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774807766423002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774807766423002, v:0}, data_checksum:2892311396, flag:1}}, init_mode_version:0, accumulate_checksum:2070800722, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774807766423002, v:0}}}}}) [2024-08-16 10:20:08.065529] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:08.065551] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:08.091504] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1723774808091500, bytes: 1496437774837, bw: 0.176540 MB/s, add_ts: 1007628, add_bytes: 186528 [2024-08-16 10:20:08.096413] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [2785][T1001_ObTimer][T1001][Y0-0000000000000000-0-0] [lt=10] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:20:08.096476] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:230) [2785][T1001_ObTimer][T1001][Y0-0000000000000000-0-0] [lt=22] [tabletchange] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12276) [2024-08-16 10:20:08.101281] INFO [LIB] stat (utility.h:1197) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=26] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=43, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:08.101309] INFO [LIB] stat (utility.h:1197) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=26] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=9, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:08.105951] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.105973] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:08.105985] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.105996] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.106051] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=42] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.106234] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8684-0-0] [lt=26] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:08.107401] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE6CD-0-0] [lt=26] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=11, avg_log_submit_to_flush_time=630, avg_log_submit_to_slide_time=1069) [2024-08-16 10:20:08.107425] INFO [PALF] read (log_cache.cpp:100) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=94] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:08.112650] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=117] new mysql sessid created(conn.sessid_=3225602748, support_ssl=false) [2024-08-16 10:20:08.112715] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=34] sm conn init succ(conn.sessid_=3225602748, sess.client_addr_="192.168.0.124:53084") [2024-08-16 10:20:08.112735] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=12] accept one succ(*s={this:0x7fa71f1dc030, session_id:3225602748, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:129}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:08.113045] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31555][sql_nio0][T1][Y0-0000000000000000-0-0] [lt=30] succeed to dispatch to tenant mysql queue(tenant_id=1) [2024-08-16 10:20:08.113701] WDIAG [COMMON] get (ob_link_hashmap.h:313) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24F-0-0] [lt=24][errcode=0] uref leak check(node->uref_=1074538691, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:08.113926] INFO [SERVER] process (obmp_connect.cpp:475) [31760][T1_MysqlQueueTh][T1][Y0-00061CCC8F47B24F-0-0] [lt=20] MySQL LOGIN(direct_client_ip="192.168.0.124", client_ip=192.168.0.124, tenant_name=sys, tenant_id=1, user_name=proxyro, host_name=%, sessid=3225602748, proxy_sessid=13882346384134438915, sess_create_time=0, from_proxy=true, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=150972045, proxy_capability=1686343, use_ssl=false, c/s protocol="OB_2_0_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:08.114289] INFO [SERVER] process (obmp_init_db.cpp:53) [4770][T1_L0_G0][T1][YB42C0A80080-00061CCCBC120252-0-0] [lt=8] init db(db_name=oceanbase) [2024-08-16 10:20:08.115561] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21123][T1_L0_G0][T1][YB42C0A80080-00061CCCC85CFAFD-0-0] [lt=5] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11047, add_timestamp:1723774808115537}) [2024-08-16 10:20:08.115599] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676709-0-0] [lt=75] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11047, locations=[{table_id:11047, partition_id:11047, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774808115580, sql_renew_time:1723774808115580, is_mark_fail:false}]) [2024-08-16 10:20:08.115665] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC9676709-0-0] [lt=64] success to process renew task(task={tenant_id:1, table_id:11047, add_timestamp:1723774808115537}, locations=[{table_id:11047, partition_id:11047, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774808115580, sql_renew_time:1723774808115580, is_mark_fail:false}]) [2024-08-16 10:20:08.117840] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=23] PNIO [ratelimit] time: 1723774808117837, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007608, add_bytes: 0 [2024-08-16 10:20:08.118958] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [21123][T1_L0_G0][T1002][YB42C0A80080-00061CCCC85CFAFD-0-0] [lt=23] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1002, table_id:11047, add_timestamp:1723774808118948}) [2024-08-16 10:20:08.118982] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670A-0-0] [lt=21] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1002, table_id=11047, locations=[{table_id:11047, partition_id:11047, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774808118971, sql_renew_time:1723774808118971, is_mark_fail:false}]) [2024-08-16 10:20:08.119003] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670A-0-0] [lt=20] success to process renew task(task={tenant_id:1002, table_id:11047, add_timestamp:1723774808118948}, locations=[{table_id:11047, partition_id:11047, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774808118971, sql_renew_time:1723774808118971, is_mark_fail:false}]) [2024-08-16 10:20:08.126985] INFO [SHARE.LOCATION] renew_cache_ (ob_tablet_ls_service.cpp:391) [21123][T1_L0_G0][T1002][YB42C0A80080-00061CCCC85CFAFD-0-0] [lt=20] [TABLET_LOCATION]success to renew tablet cache(tablet_cache={cache_key:{tenant_id:1002, tablet_id:{id:200569}}, ls_id:{id:1026}, renew_time:1723774808126895, transfer_seq:3}) [2024-08-16 10:20:08.140918] INFO [STORAGE.BLKMGR] defragment (ob_shared_macro_block_manager.cpp:445) [31740][T1_SSTableDefra][T1][Y0-0000000000000000-0-0] [lt=5] skip defragment task(ret=0, need_defragment=false) [2024-08-16 10:20:08.140946] INFO [STORAGE.BLKMGR] defragment (ob_shared_macro_block_manager.cpp:485) [31740][T1_SSTableDefra][T1][Y0-0000000000000000-0-0] [lt=25] successfully defragment data blocks(ret=0, rewrite_cnt=0, block_used_size_.count()=1) [2024-08-16 10:20:08.140959] INFO [STORAGE.BLKMGR] defragment (ob_shared_macro_block_manager.cpp:503) [31740][T1_SSTableDefra][T1][Y0-0000000000000000-0-0] [lt=12] successfully defragment data blocks(rewrite_cnt=0) [2024-08-16 10:20:08.150275] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:08.150393] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:08.150431] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=31] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1220) [2024-08-16 10:20:08.150458] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:08.150472] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:08.150480] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:08.150887] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:08.151216] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=36] Wash time detail, (compute_wash_size_time=188, refresh_score_time=67, wash_time=16) [2024-08-16 10:20:08.151605] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:08.151621] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:08.152333] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7E2-0-0] [lt=79][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:08.152866] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7E2-0-0] [lt=20] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:08.154562] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=94, clean_time=0, clean_start_pos=817882, clean_num=62914) [2024-08-16 10:20:08.157240] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.157270] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:20:08.157283] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.157295] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.157317] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.158858] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B51-0-0] [lt=23] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:08.158881] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B51-0-0] [lt=21] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:08.159431] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:08.159936] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74485-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=9, avg_log_submit_to_flush_time=587, avg_log_submit_to_slide_time=974) [2024-08-16 10:20:08.159968] INFO [PALF] read (log_cache.cpp:100) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:08.165721] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=16] ====== tenant freeze timer task ====== [2024-08-16 10:20:08.184104] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=25] gc stale ls task succ [2024-08-16 10:20:08.200508] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E30-0-0] [lt=23][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=10, len=121, pos=108, bitmap=NULL) [2024-08-16 10:20:08.201206] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF227-0-0] [lt=22][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509933, log_task={header:{begin_lsn:{lsn:18544232939}, end_lsn:{lsn:18544233061}, log_id:53509933, min_scn:{val:1723774807889740002, v:0}, max_scn:{val:1723774807889740002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544232817}, prev_proposal_id:7, committed_end_lsn:{lsn:18544232939}, data_checksum:1913811101, accum_checksum:2431372622, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774808101136, freeze_ts:1723774808101137, submit_ts:1723774808101147, flushed_ts:1723774808101922, gen_to_freeze cost time:1, gen_to_submit cost time:11, submit_to_flush cost time:775}, fs_cb_begin_ts=1723774808201190, log_life_time=100054) [2024-08-16 10:20:08.206003] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.206025] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:08.206037] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.206049] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.206065] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.213504] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [2786][T1001_TabletShe][T1001][Y0-0000000000000000-0-0] [lt=11] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:20:08.213535] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:101) [2786][T1001_TabletShe][T1001][Y0-0000000000000000-0-0] [lt=24] [emptytablet] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12276) [2024-08-16 10:20:08.221568] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [2549][T1002_LCLSender][T1002][Y0-0000000000000000-0-0] [lt=72] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=1135, total_destructed_detector=1135, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x7fa7110382b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-08-16 10:20:08.236559] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=9] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:08.236568] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:08.236622] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=16] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:08.236701] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:08.236979] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=23] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:08.237092] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:08.237585] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:08.242712] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=23] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=7) [2024-08-16 10:20:08.250344] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:08.250472] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:08.250489] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:08.250497] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:08.251687] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:08.257314] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.257334] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:08.257345] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.257356] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.257370] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.270545] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=18] current monitor number(seq_=-1) [2024-08-16 10:20:08.272325] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AB5-0-0] [lt=16] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:08.272363] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AB5-0-0] [lt=21] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774807766423004, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774807766423004, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:08.300987] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=17] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:20:08.301015] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=25] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=1, size_used=5307, mem_used=39515136) [2024-08-16 10:20:08.301896] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=44, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:20:08.306107] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.306130] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:08.306141] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.306152] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.306171] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.306365] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8688-0-0] [lt=43] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:08.306928] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=548, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:08.306973] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=41] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:08.307011] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=32, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:08.307087] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=48] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=35, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:08.307406] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE6D3-0-0] [lt=20] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:08.307428] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE6D3-0-0] [lt=21] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1050, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:08.310591] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=22] ====== tenant freeze timer task ====== [2024-08-16 10:20:08.310634] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=32][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:08.350419] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:08.350534] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:08.350550] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:08.350561] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:08.351764] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=33][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:08.355066] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:08.355354] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=52] Wash time detail, (compute_wash_size_time=140, refresh_score_time=76, wash_time=16) [2024-08-16 10:20:08.357388] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.357409] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:08.357419] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.357429] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.357443] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.358486] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=23] Cache wash clean map node details(ret=0, clean_node_count=89, clean_time=0, clean_start_pos=880796, clean_num=62914) [2024-08-16 10:20:08.384930] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=24] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:08.401209] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF233-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1501, avg_group_log_size=150) [2024-08-16 10:20:08.401256] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF233-0-0] [lt=28][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509935, log_task={header:{begin_lsn:{lsn:18544233183}, end_lsn:{lsn:18544233305}, log_id:53509935, min_scn:{val:1723774808265224000, v:0}, max_scn:{val:1723774808265224000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544233061}, prev_proposal_id:7, committed_end_lsn:{lsn:18544233183}, data_checksum:1913811101, accum_checksum:2170859564, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774808301206, freeze_ts:1723774808301207, submit_ts:1723774808301215, flushed_ts:1723774808301842, gen_to_freeze cost time:1, gen_to_submit cost time:9, submit_to_flush cost time:627}, fs_cb_begin_ts=1723774808401244, log_life_time=100038) [2024-08-16 10:20:08.406091] INFO pn_ratelimit (group.c:551) [31557][IngressService][T0][Y0-0000000000000000-0-0] [lt=20] PNIO set ratelimit as 9223372036854775807 bytes/s, grp_id=2 [2024-08-16 10:20:08.406189] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.406218] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=27] handle task succ(id={id:1260}) [2024-08-16 10:20:08.406235] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.406248] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.406273] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=15] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.424338] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE87-0-0] [lt=24] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:08.424388] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE87-0-0] [lt=30] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774807909136004, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774807909136004, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:08.424890] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=26] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574652, offset=42150293}, end_cursor:ObLogCursor{file_id=65, log_id=18574653, offset=42150534}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516d0050, len:1643, log_data_len:241, seq:18574652, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:08.425601] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=42] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574653, offset=42150534}, end_cursor:ObLogCursor{file_id=65, log_id=18574654, offset=42150783}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751708050, len:1402, log_data_len:249, seq:18574653, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:08.426205] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=50] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:08.426221] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=21] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:08.427106] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCAA-0-0] [lt=26] push gts task success(*task={:""}) [2024-08-16 10:20:08.431427] INFO [STORAGE.TRANS] alloc (ob_ts_response_handler.cpp:108) [31439][pnio1][T0][YB42C0A80080-00061CCCC807BCAA-0-0] [lt=41] ts response task statistics(alloc_count=170000802, free_count=85000400) [2024-08-16 10:20:08.432108] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=23] gts statistics(tenant_id=1, gts_rpc_cnt=16, get_gts_cache_cnt=104, get_gts_with_stc_cnt=6, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-08-16 10:20:08.436220] INFO [SHARE.LOCATION] renew_all_ls_locations_by_rpc (ob_ls_location_service.cpp:783) [31503][AutoLSLocRpc][T0][YB42C0A80080-00061CCCC718F0C0-0-0] [lt=91] [LS_LOCATION] Get ls leaders by RPC(ret=0, ret="OB_SUCCESS", dests=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882", "192.168.0.124:2882", "192.168.0.125:2882", "192.168.0.127:2882", "192.168.0.128:2882", "192.168.0.131:2882", "192.168.0.133:2882"], leaders=[{key:{tenant_id:1, ls_id:{id:1}, cluster_id:1713182723}, location:{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:11}}, {key:{tenant_id:1001, ls_id:{id:1}, cluster_id:1713182723}, location:{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:7}}, {key:{tenant_id:1002, ls_id:{id:1024}, cluster_id:1713182723}, location:{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:47}}, {key:{tenant_id:1002, ls_id:{id:1}, cluster_id:1713182723}, location:{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:7}}, {key:{tenant_id:1002, ls_id:{id:1079}, cluster_id:1713182723}, location:{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:7}}, {key:{tenant_id:1002, ls_id:{id:1261}, cluster_id:1713182723}, location:{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:5}}, {key:{tenant_id:1002, ls_id:{id:1028}, cluster_id:1713182723}, location:{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:6}}, {key:{tenant_id:1002, ls_id:{id:1260}, cluster_id:1713182723}, location:{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:29}}, {key:{tenant_id:1002, ls_id:{id:1026}, cluster_id:1713182723}, location:{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:20}}, {key:{tenant_id:1002, ls_id:{id:1262}, cluster_id:1713182723}, location:{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}}]) [2024-08-16 10:20:08.440982] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCAA-0-0] [lt=20] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:08.442818] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=23] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=6) [2024-08-16 10:20:08.443538] INFO [STORAGE.TRANS] process (ob_trans_rpc.cpp:142) [2809][T1001_L0_G0][T1001][YB42C0A8007C-000616D3F7C3A864-0-0] [lt=19] transaction rpc statistics(total_rt=328, total_process=10, avg_rt=29) [2024-08-16 10:20:08.444267] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=21] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574654, offset=42150783}, end_cursor:ObLogCursor{file_id=65, log_id=18574655, offset=42151032}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751764050, len:1153, log_data_len:249, seq:18574654, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:08.444818] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=45] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574655, offset=42151032}, end_cursor:ObLogCursor{file_id=65, log_id=18574656, offset=42151281}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751778050, len:904, log_data_len:249, seq:18574655, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:08.450494] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:08.450620] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:08.450636] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:08.450643] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:08.451155] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=15] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=634, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:08.451194] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=40, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:08.451227] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=17] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=15, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:08.451680] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE6D7-0-0] [lt=33] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:08.451702] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE6D7-0-0] [lt=20] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1206, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:08.451838] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:08.457475] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.457499] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:20:08.457509] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.457518] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.457532] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.506292] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.506332] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=37] handle task succ(id={id:1260}) [2024-08-16 10:20:08.506346] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.506368] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.506384] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.506577] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD868C-0-0] [lt=34] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:08.526269] INFO [CLOG] is_in_sync (ob_log_handler.cpp:546) [4774][T1_L0_G0][T1002][YB42C0A80083-000616E44D0F990F-0-0] [lt=12] is_in_sync(ret=0, id=1079, is_log_sync=true, is_need_rebuild=false) [2024-08-16 10:20:08.526406] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [4774][T1_L0_G0][T1][YB42C0A80083-000616E44D0F990F-0-0] [lt=46][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:08.526674] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007D-00061CCCA5024D1F-0-0] [lt=36][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:08.526897] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=41] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=1, size_used=7044, mem_used=12478464) [2024-08-16 10:20:08.534349] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=33] start do ls ha handler(ls_id_array_=[{id:1079}, {id:1260}, {id:1262}, {id:1261}]) [2024-08-16 10:20:08.550587] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:08.550702] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:08.550721] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:08.550728] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:08.551932] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:08.557546] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=25] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.557567] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:08.557578] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.557588] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.557602] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.558937] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:08.559210] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B59-0-0] [lt=28] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:08.559262] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash time detail, (compute_wash_size_time=145, refresh_score_time=79, wash_time=21) [2024-08-16 10:20:08.562538] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=81, clean_time=0, clean_start_pos=943710, clean_num=62914) [2024-08-16 10:20:08.562781] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [31763][T1_LCLSender][T1][Y0-0000000000000000-0-0] [lt=39] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=1135, total_destructed_detector=1135, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x7fa76bdfe2b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-08-16 10:20:08.582176] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=23] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 390.00) [2024-08-16 10:20:08.582203] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=22] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:08.582269] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=29] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.99, rt: 466.50) [2024-08-16 10:20:08.582285] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=13] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:08.582304] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=13] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582319] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=13] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582326] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582333] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582342] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582349] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582354] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582358] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582365] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582382] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582391] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582401] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582418] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=16] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582425] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582436] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.582441] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:08.591100] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [31738][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=56] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:940, allocator used:3737440, allocator total:3859072}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:140, allocator used:9182740, allocator total:9182740}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=13041812, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:63, allocator used:124992, allocator total:130816}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:08.591696] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=19][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:08.594035] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=22] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:08.594162] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=22][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:08.594192] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=27][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:08.594221] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=27] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:08.601434] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF23F-0-0] [lt=22] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1623) [2024-08-16 10:20:08.601474] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF23F-0-0] [lt=21] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=13, avg_log_submit_to_flush_time=693, avg_log_submit_to_slide_time=91013) [2024-08-16 10:20:08.601491] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF23F-0-0] [lt=15][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509937, log_task={header:{begin_lsn:{lsn:18544233427}, end_lsn:{lsn:18544233549}, log_id:53509937, min_scn:{val:1723774808390517001, v:0}, max_scn:{val:1723774808390517001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544233305}, prev_proposal_id:7, committed_end_lsn:{lsn:18544233427}, data_checksum:1913811101, accum_checksum:1264746402, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774808501386, freeze_ts:1723774808501387, submit_ts:1723774808501397, flushed_ts:1723774808502133, gen_to_freeze cost time:1, gen_to_submit cost time:11, submit_to_flush cost time:736}, fs_cb_begin_ts=1723774808601460, log_life_time=100074) [2024-08-16 10:20:08.601521] INFO [PALF] read (log_cache.cpp:100) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1623, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:08.602064] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=21, stat_interval=1000000, avg (count/cost)=34, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:08.606368] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=15] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.606391] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:08.606401] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.606411] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.606426] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.607887] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=17] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:08.618490] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=23] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60744, mem_used=137263104) [2024-08-16 10:20:08.642917] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=22] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=5) [2024-08-16 10:20:08.650672] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:08.650768] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:08.650785] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:08.650792] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:08.652048] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=39][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:08.657626] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.657647] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:08.657657] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.657667] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.657678] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.659728] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=41] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=530, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:08.659808] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=30] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=36, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:08.660167] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74494-0-0] [lt=45] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=13, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:08.660184] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA74494-0-0] [lt=16] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=999, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:08.685738] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=27] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:08.706439] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.706462] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:08.706474] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.706484] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.706501] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.750755] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:08.750858] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=47, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:08.750905] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:08.750917] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:08.750930] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:08.750944] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:08.752031] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE6E0-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=5, avg_log_gen_to_submit_time=68, avg_log_submit_to_flush_time=706, avg_log_submit_to_slide_time=1223) [2024-08-16 10:20:08.752103] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=12][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:08.757699] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.757720] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:08.757730] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.757740] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.757759] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.763051] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=19] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:08.763369] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=46] Wash time detail, (compute_wash_size_time=153, refresh_score_time=73, wash_time=24) [2024-08-16 10:20:08.767041] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=16] Cache wash clean map node details(ret=0, clean_node_count=89, clean_time=0, clean_start_pos=1006624, clean_num=62914) [2024-08-16 10:20:08.801479] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF24B-0-0] [lt=31] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:08.801536] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF24B-0-0] [lt=31] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:08.801545] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF24B-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=100097, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:08.801551] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF24B-0-0] [lt=4][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509939, log_task={header:{begin_lsn:{lsn:18544233671}, end_lsn:{lsn:18544233793}, log_id:53509939, min_scn:{val:1723774808390517003, v:0}, max_scn:{val:1723774808390517003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544233549}, prev_proposal_id:7, committed_end_lsn:{lsn:18544233671}, data_checksum:1913811101, accum_checksum:3816273310, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774808701367, freeze_ts:1723774808701367, submit_ts:1723774808701374, flushed_ts:1723774808702115, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:741}, fs_cb_begin_ts=1723774808801529, log_life_time=100162) [2024-08-16 10:20:08.806516] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.806539] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:08.806550] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.806560] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.806573] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.843031] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=24] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=4) [2024-08-16 10:20:08.850814] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:08.850952] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:08.850969] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:08.850977] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:08.851677] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:08.852181] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:08.857775] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.857807] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=29] handle task succ(id={id:1260}) [2024-08-16 10:20:08.857819] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.857831] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.857853] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.878967] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [31754][T1_LockWaitMgr][T1][Y0-0000000000000000-0-0] [lt=24] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-08-16 10:20:08.883890] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1623) [2024-08-16 10:20:08.902152] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=31] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=619, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:08.906605] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=4] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.906650] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=42] handle task succ(id={id:1260}) [2024-08-16 10:20:08.906685] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=31] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.906711] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.906737] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.932677] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=22] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:08.950887] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:08.951008] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:08.951028] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:08.951064] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=33] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:08.952255] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:08.957846] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:08.957868] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:08.957878] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:08.957890] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:08.957910] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:20:08.967590] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=28] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:08.967873] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash time detail, (compute_wash_size_time=117, refresh_score_time=79, wash_time=16) [2024-08-16 10:20:08.971292] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=31] Cache wash clean map node details(ret=0, clean_node_count=112, clean_time=0, clean_start_pos=1069538, clean_num=62914) [2024-08-16 10:20:08.998217] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:08.998266] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=28] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:08.998416] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:08.998438] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=17][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:08.998457] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=15][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:08.998501] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=41][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:09.001644] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF257-0-0] [lt=34][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509942, log_task={header:{begin_lsn:{lsn:18544234318}, end_lsn:{lsn:18544234440}, log_id:53509942, min_scn:{val:1723774808890582000, v:0}, max_scn:{val:1723774808890582000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544233915}, prev_proposal_id:7, committed_end_lsn:{lsn:18544234318}, data_checksum:1913811101, accum_checksum:750044045, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774808901484, freeze_ts:1723774808901484, submit_ts:1723774808901488, flushed_ts:1723774808902206, gen_to_freeze cost time:0, gen_to_submit cost time:4, submit_to_flush cost time:718}, fs_cb_begin_ts=1723774809001629, log_life_time=100145) [2024-08-16 10:20:09.006692] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.006733] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=38] handle task succ(id={id:1260}) [2024-08-16 10:20:09.006750] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.006777] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.006801] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.012935] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=20] gc stale ls task succ [2024-08-16 10:20:09.013006] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=4] gc stale ls task succ [2024-08-16 10:20:09.013039] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=22] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:09.013055] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=10] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:09.013061] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=5] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:09.017129] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:09.017211] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=27] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:09.031999] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C2F-0-0] [lt=54] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774809031967) [2024-08-16 10:20:09.043169] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=57] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=3) [2024-08-16 10:20:09.050961] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=36] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:09.051083] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:09.051100] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:09.051109] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:09.052332] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:09.057929] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=34] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.057959] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=28] handle task succ(id={id:1260}) [2024-08-16 10:20:09.057969] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.057978] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.058010] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.059106] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=22] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:09.063357] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=20] new mysql sessid created(conn.sessid_=3225498490, support_ssl=false) [2024-08-16 10:20:09.063399] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=20] sm conn init succ(conn.sessid_=3225498490, sess.client_addr_="192.168.0.124:53088") [2024-08-16 10:20:09.063421] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=15] accept one succ(*s={this:0x7fa71f01e030, session_id:3225498490, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:155}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:09.063685] INFO [SERVER] dispatch_req (ob_srv_deliver.cpp:275) [31555][sql_nio0][T1002][Y0-0000000000000000-0-0] [lt=22] succeed to dispatch to tenant mysql queue(tenant_id=1002) [2024-08-16 10:20:09.063775] WDIAG [COMMON] get (ob_link_hashmap.h:313) [2542][T1002_MysqlQueu][T1002][Y0-00061CCC96923581-0-0] [lt=47][errcode=0] uref leak check(node->uref_=1073901798, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe1402ed 0xa1f3da2 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:09.064279] WDIAG [COMMON] get (ob_link_hashmap.h:313) [2542][T1002_MysqlQueu][T1002][Y0-00061CCC96923581-0-0] [lt=19][errcode=0] uref leak check(node->uref_=1073901798, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13de28 0xe115aaa 0xa1fa544 0xa1f405a 0x4e26201 0x138ce0d2 0x138cf32a 0x138cf8a9 0x12d30651 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:09.064550] INFO [SERVER] process (obmp_connect.cpp:475) [2542][T1002_MysqlQueu][T1002][Y0-00061CCC96923581-0-0] [lt=19] MySQL LOGIN(direct_client_ip="192.168.0.124", client_ip=192.168.6.97, tenant_name=epean, tenant_id=1002, user_name=epean_trade_test, host_name=%, sessid=3225498490, proxy_sessid=13882346384134439129, sess_create_time=0, from_proxy=true, from_java_client=false, from_oci_client=false, from_jdbc_client=false, capability=683647754, proxy_capability=1686343, use_ssl=false, c/s protocol="OB_2_0_CS_TYPE", autocommit=true, proc_ret=0, ret=0, conn->client_type_=3, conn->client_version_=0) [2024-08-16 10:20:09.065018] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [2663][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=60] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,1,False,{False,},False,False,True}]) [2024-08-16 10:20:09.065612] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=36][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:09.065646] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=33][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:09.067064] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [31642][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=27] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-08-16 10:20:09.071175] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E096E-0-0] [lt=19] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1002, table_id:10006, add_timestamp:1723774809071160}) [2024-08-16 10:20:09.071222] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670B-0-0] [lt=10] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1002, table_id=10006, locations=[{table_id:10006, partition_id:10006, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809071203, sql_renew_time:1723774809071203, is_mark_fail:false}]) [2024-08-16 10:20:09.071261] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670B-0-0] [lt=37] success to process renew task(task={tenant_id:1002, table_id:10006, add_timestamp:1723774809071160}, locations=[{table_id:10006, partition_id:10006, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809071203, sql_renew_time:1723774809071203, is_mark_fail:false}]) [2024-08-16 10:20:09.071437] WDIAG [SHARE] inner_to_show_str (ob_system_variable.cpp:515) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E096E-0-0] [lt=20][errcode=0] sys var casted obj is not ObVarcharType(ret=0, value={"NULL":"NULL"}, *res_obj={"NULL":"NULL"}, get_name()=secure_file_priv) [2024-08-16 10:20:09.075574] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E096F-0-0] [lt=22] succeed to calculate base table sharding info(table_id=18446744073709551614, ref_table_id=104, is_global_index=false) [2024-08-16 10:20:09.075619] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E096F-0-0] [lt=21] total rowcount, use statistics(table_meta_info_.table_row_count_=14, table_meta_info_.average_row_size_=2.020000000000000000e+02, table_meta_info_.micro_block_count_=1, table_meta_info_.part_size_=2.828000000000000000e+03) [2024-08-16 10:20:09.089397] WDIAG [SQL.PC] common_free (ob_lib_cache_object_manager.cpp:141) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E096F-0-0] [lt=15][errcode=0] set logical del time(cache_obj->get_logical_del_time()=10451121348313, cache_obj->added_lc()=false, cache_obj->get_object_id()=254222, cache_obj->get_tenant_id()=1002, lbt()="0x12a0688c 0xaa7c9e4 0x4e44d8a 0x4e36df8 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:09.090998] WDIAG [SERVER] disconnect (obmp_packet_sender.cpp:830) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E0970-0-0] [lt=23][errcode=0] server close connection(sessid=3225498490, proxy_sessid=13882346384134439129, stack="0x12a0688c 0xa219970 0xa1e504b 0xa527c01 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:09.091031] INFO [SERVER] process (obmp_quit.h:56) [12976][T1002_L0_G0][T1002][YB42C0A80080-00061CCCC52E0970-0-0] [lt=29] quit [2024-08-16 10:20:09.091068] WDIAG [RPC.OBMYSQL] push_close_req (ob_sql_nio.cpp:824) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=15][errcode=-4015] close sql sock by user req(*s={this:0x7fa71f01e030, session_id:3225498490, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:true, reader:{fd:155}, err:5, last_decode_time:1723774809090651, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:1097, pending_flag:1, may_handling_flag:true, handler_close_flag:false}) [2024-08-16 10:20:09.091124] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=48] kill and revert session(conn.sessid_=3225498490, proxy_sessid=13882346384134439129, server_id=16, ret=0) [2024-08-16 10:20:09.091135] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=10] can close safely, do destroy(*s={this:0x7fa71f01e030, session_id:3225498490, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:true, reader:{fd:155}, err:5, last_decode_time:1723774809090651, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:1097, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-08-16 10:20:09.091156] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=10] unlock session of tenant(conn.sessid_=3225498490, proxy_sessid=13882346384134439129, conn.tenant_id_=1002) [2024-08-16 10:20:09.091193] WDIAG [COMMON] get (ob_link_hashmap.h:313) [12976][T1002_L0_G0][T1002][Y0-0000000000000000-0-0] [lt=11][errcode=0] uref leak check(node->uref_=1073901798, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x7fa776417ea5 0x7fa77614096d") [2024-08-16 10:20:09.091234] INFO [SERVER] run (obmp_disconnect.cpp:82) [12976][T1002_L0_G0][T1002][Y0-0000000000000000-0-0] [lt=18] free session successfully(sessid=3225498490) [2024-08-16 10:20:09.091248] INFO [SERVER] run (obmp_disconnect.cpp:89) [12976][T1002_L0_G0][T1002][Y0-0000000000000000-0-0] [lt=13] mark session id unused(sessid=3225498490) [2024-08-16 10:20:09.091169] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [31555][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=12] connection close(sessid=3225498490, proxy_sessid=13882346384134439129, tenant_id=1002, server_id=16, from_proxy=true, from_java_client=false, c/s protocol="OB_2_0_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB42C0A80080-00061CCCC52E096F-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:29, cur_pkt_pos_:32, pkt_rec[0]:{send:obp_mysql_header_{len_:6413, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{send:obp20_header_:{payload_len:53, pkt_seq:0, req_id:2361247, flag_.flags:43, ext_flags:0}, obp_mysql_header_{len_:575, seq_:2}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[2]:{send:obp20_header_:{payload_len:19, pkt_seq:0, req_id:2361248, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:504, seq_:2}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{receive:obp20_header_:{payload_len:160, pkt_seq:0, req_id:2361249, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:156, rec_:160, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[4]:{send:obp_compess_header_{com_len:364, com_seq:1}, obp20_header_:{payload_len:336, pkt_seq:1, req_id:2361249, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_compess_header_{com_len:364, com_seq:1}, obp20_header_:{payload_len:336, pkt_seq:1, req_id:2361249, flag_.flags:10, ext_flags:0}, obp_mysql_header_{pkt_num_:6, seq_:8}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[6]:{send:obp_compess_header_{com_len:364, com_seq:1}, obp20_header_:{payload_len:336, pkt_seq:1, req_id:2361249, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:9, seq_:9}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[7]:{send:obp_compess_header_{com_len:364, com_seq:1}, obp20_header_:{payload_len:336, pkt_seq:1, req_id:2361249, flag_.flags:10, ext_flags:0}, obp_mysql_header_{pkt_num_:1, seq_:10}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[8]:{send:obp_compess_header_{com_len:364, com_seq:1}, obp20_header_:{payload_len:336, pkt_seq:1, req_id:2361249, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:9, seq_:11}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[9]:{send:obp_compess_header_{com_len:364, com_seq:1}, obp20_header_:{payload_len:336, pkt_seq:1, req_id:2361249, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:12, seq_:12}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[10]:{receive:obp20_header_:{payload_len:62, pkt_seq:0, req_id:2361250, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:58, rec_:62, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[11]:{send:obp_compess_header_{com_len:333, com_seq:1}, obp20_header_:{payload_len:305, pkt_seq:1, req_id:2361250, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[12]:{send:obp_compess_header_{com_len:333, com_seq:1}, obp20_header_:{payload_len:305, pkt_seq:1, req_id:2361250, flag_.flags:10, ext_flags:0}, obp_mysql_header_{pkt_num_:3, seq_:5}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[13]:{send:obp_compess_header_{com_len:333, com_seq:1}, obp20_header_:{payload_len:305, pkt_seq:1, req_id:2361250, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:9, seq_:6}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[14]:{send:obp_compess_header_{com_len:333, com_seq:1}, obp20_header_:{payload_len:305, pkt_seq:1, req_id:2361250, flag_.flags:10, ext_flags:0}, obp_mysql_header_{pkt_num_:1, seq_:7}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[15]:{send:obp_compess_header_{com_len:333, com_seq:1}, obp20_header_:{payload_len:305, pkt_seq:1, req_id:2361250, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:9, seq_:8}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[16]:{send:obp_compess_header_{com_len:333, com_seq:1}, obp20_header_:{payload_len:305, pkt_seq:1, req_id:2361250, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:12, seq_:9}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[17]:{receive:obp20_header_:{payload_len:42, pkt_seq:0, req_id:2361251, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:38, rec_:42, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[18]:{send:obp_compess_header_{com_len:340, com_seq:1}, obp20_header_:{payload_len:312, pkt_seq:1, req_id:2361251, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[19]:{send:obp_compess_header_{com_len:340, com_seq:1}, obp20_header_:{payload_len:312, pkt_seq:1, req_id:2361251, flag_.flags:10, ext_flags:0}, obp_mysql_header_{pkt_num_:2, seq_:4}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[20]:{send:obp_compess_header_{com_len:340, com_seq:1}, obp20_header_:{payload_len:312, pkt_seq:1, req_id:2361251, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:9, seq_:5}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[21]:{send:obp_compess_header_{com_len:340, com_seq:1}, obp20_header_:{payload_len:312, pkt_seq:1, req_id:2361251, flag_.flags:10, ext_flags:0}, obp_mysql_header_{pkt_num_:1, seq_:6}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[22]:{send:obp_compess_header_{com_len:340, com_seq:1}, obp20_header_:{payload_len:312, pkt_seq:1, req_id:2361251, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:9, seq_:7}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[23]:{send:obp_compess_header_{com_len:340, com_seq:1}, obp20_header_:{payload_len:312, pkt_seq:1, req_id:2361251, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:12, seq_:8}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[24]:{receive:obp20_header_:{payload_len:19, pkt_seq:0, req_id:2361252, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:15, rec_:19, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[25]:{send:obp_compess_header_{com_len:331, com_seq:1}, obp20_header_:{payload_len:303, pkt_seq:1, req_id:2361252, flag_.flags:8, ext_flags:0}, obp_mysql_header_{len_:5, seq_:2}, pkt_name:"PKT_RESHEAD", obp_mysql_header_.is_send_:1}, pkt_rec[26]:{send:obp_compess_header_{com_len:331, com_seq:1}, obp20_header_:{payload_len:303, pkt_seq:1, req_id:2361252, flag_.flags:8, ext_flags:0}, obp_mysql_header_{pkt_num_:1, seq_:3}, pkt_name:"PKT_FIELD", obp_mysql_header_.is_send_:1}, pkt_rec[27]:{send:obp_compess_header_{com_len:331, com_seq:1}, obp20_header_:{payload_len:303, pkt_seq:1, req_id:2361252, flag_.flags:8, ext_flags:0}, obp_mysql_header_{len_:9, seq_:4}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[28]:{send:obp_compess_header_{com_len:331, com_seq:1}, obp20_header_:{payload_len:303, pkt_seq:1, req_id:2361252, flag_.flags:8, ext_flags:0}, obp_mysql_header_{pkt_num_:13, seq_:17}, pkt_name:"PKT_ROW", obp_mysql_header_.is_send_:1}, pkt_rec[29]:{send:obp_compess_header_{com_len:465, com_seq:2}, obp20_header_:{payload_len:437, pkt_seq:2, req_id:2361252, flag_.flags:11, ext_flags:0}, obp_mysql_header_{len_:393, seq_:18}, pkt_name:"PKT_EOF", obp_mysql_header_.is_send_:1}, pkt_rec[30]:{send:obp_compess_header_{com_len:465, com_seq:2}, obp20_header_:{payload_len:437, pkt_seq:2, req_id:2361252, flag_.flags:11, ext_flags:0}, obp_mysql_header_{len_:44, seq_:19}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[31]:{receive:obp20_header_:{payload_len:5, pkt_seq:0, req_id:2361253, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:1, rec_:5, seq_:0}, pkt_name:"COM_QUIT", obp_mysql_header_.type_:1, obp_mysql_header_.is_send_:0}], disconnect_state=3) [2024-08-16 10:20:09.099100] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=25] PNIO [ratelimit] time: 1723774809099098, bytes: 1496437894664, bw: 0.113414 MB/s, add_ts: 1007598, add_bytes: 119827 [2024-08-16 10:20:09.101756] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF25D-0-0] [lt=27][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509943, log_task={header:{begin_lsn:{lsn:18544234440}, end_lsn:{lsn:18544234562}, log_id:53509943, min_scn:{val:1723774808890582001, v:0}, max_scn:{val:1723774808890582001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544234318}, prev_proposal_id:7, committed_end_lsn:{lsn:18544234440}, data_checksum:1913811101, accum_checksum:1978359016, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774809001599, freeze_ts:1723774809001599, submit_ts:1723774809001607, flushed_ts:1723774809002341, gen_to_freeze cost time:0, gen_to_submit cost time:8, submit_to_flush cost time:734}, fs_cb_begin_ts=1723774809101742, log_life_time=100143) [2024-08-16 10:20:09.101865] INFO [LIB] stat (utility.h:1197) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=27] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=37, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:09.101898] INFO [LIB] stat (utility.h:1197) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=31] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=31, stat_interval=1000000, avg (count/cost)=6, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:09.106259] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80085-00061CCD1E636C15-0-0] [lt=19][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:09.106779] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.106799] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:09.106810] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.106819] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.106833] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.106947] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD8698-0-0] [lt=46] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:09.108035] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA744A3-0-0] [lt=21] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=7, avg_log_submit_to_flush_time=601, avg_log_submit_to_slide_time=1002) [2024-08-16 10:20:09.108056] INFO [PALF] read (log_cache.cpp:100) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:09.125472] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=28] PNIO [ratelimit] time: 1723774809125469, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007632, add_bytes: 0 [2024-08-16 10:20:09.151040] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:09.151184] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:09.151223] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=34] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1220) [2024-08-16 10:20:09.151242] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:09.151249] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:09.151253] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:09.151290] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2975][T1002_DupLoop][T1002][Y0-0000000000000000-0-0] [lt=96] get_max_applied_scn(scn={val:18446744073709551615, v:3}, this={ls_id_:{id:1079}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:18446744073709551615, v:3}}) [2024-08-16 10:20:09.151847] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:09.152377] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:134) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=24] ObStandbyTimestampService state(*this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:20:09.152412] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=33][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:09.155433] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7E3-0-0] [lt=70][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:09.155999] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7E3-0-0] [lt=40] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:09.158001] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.158021] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:09.158031] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.158042] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.158056] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.159626] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B65-0-0] [lt=18] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:09.159655] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B65-0-0] [lt=26] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:09.160195] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:09.160585] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F736BB-0-0] [lt=13] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=8, avg_log_submit_to_flush_time=593, avg_log_submit_to_slide_time=983) [2024-08-16 10:20:09.160611] INFO [PALF] read (log_cache.cpp:100) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=51] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:09.171724] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=21] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:09.171776] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2747][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=24] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:20:09.172004] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=33] Wash time detail, (compute_wash_size_time=123, refresh_score_time=65, wash_time=16) [2024-08-16 10:20:09.175481] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Cache wash clean map node details(ret=0, clean_node_count=86, clean_time=0, clean_start_pos=1132452, clean_num=62914) [2024-08-16 10:20:09.184150] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=38] gc stale ls task succ [2024-08-16 10:20:09.201729] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E31-0-0] [lt=28] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11031, add_timestamp:1723774809201706}) [2024-08-16 10:20:09.201762] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670C-0-0] [lt=22] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11031, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:09.202259] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E31-0-0] [lt=30][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=70, len=321, pos=292, bitmap=NULL) [2024-08-16 10:20:09.202266] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670C-0-0] [lt=21] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11031, locations=[{table_id:11031, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:09.202316] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670C-0-0] [lt=49] success to process renew task(task={tenant_id:1, table_id:11031, add_timestamp:1723774809201706}, locations=[{table_id:11031, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774809202242, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:09.206841] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.206863] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:09.206874] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.206886] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.206907] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.208240] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCC998AE0AC-0-0] [lt=17] receive register_req from acceptor, ignore(self="192.168.0.128:2882", palf_id=1260, child={server:"192.168.0.125:2882", region:"sys_region", register_time_us:1723774809207210, keepalive_ts:-1}) [2024-08-16 10:20:09.208266] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCC998AE0AC-0-0] [lt=25] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1260, self="192.168.0.128:2882", child={server:"192.168.0.125:2882", region:"sys_region", register_time_us:1723774809207210, keepalive_ts:-1}, is_to_leader=true) [2024-08-16 10:20:09.236470] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=24] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:09.236621] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=15] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:09.236662] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:09.236805] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:09.236975] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=26] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:09.236997] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:09.237009] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=17] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:09.238804] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007E-00061EFF757F6EB0-0-0] [lt=21][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:09.243293] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=22] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=2) [2024-08-16 10:20:09.243848] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80086-00061EFF757FAF15-0-0] [lt=18][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:09.248027] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD16DAE258-0-0] [lt=11] receive register_req from acceptor, ignore(self="192.168.0.128:2882", palf_id=1260, child={server:"192.168.0.133:2882", region:"sys_region", register_time_us:1723774809245845, keepalive_ts:-1}) [2024-08-16 10:20:09.248053] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD16DAE258-0-0] [lt=24] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1260, self="192.168.0.128:2882", child={server:"192.168.0.133:2882", region:"sys_region", register_time_us:1723774809245845, keepalive_ts:-1}, is_to_leader=true) [2024-08-16 10:20:09.251110] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:09.251212] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:09.251228] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:09.251234] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:09.252450] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:09.254388] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2752][T1001_TenantMet][T1001][Y0-0000000000000000-0-0] [lt=51] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=0, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:09.258104] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.258125] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:09.258136] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.258145] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.258159] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.271421] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=23] current clock is (clock_=0, max_seq_=0, mem_limit=1643328294, cur_mem_hold=289406976, attr_.tenant_id_=1002) [2024-08-16 10:20:09.271450] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=23] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1723774780140540, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1723774809270835, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-08-16 10:20:09.276123] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [2527][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=25] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1723774779228511, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1723774809275138, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-08-16 10:20:09.278595] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=52] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1723774790147030, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1723774809278337, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-08-16 10:20:09.279051] INFO [SQL.PC] inner_cache_evict (ob_ps_cache.cpp:788) [17389][T1002_PsCacheEv][T1002][Y0-0000000000000000-0-0] [lt=29] ps cache evict(tenant_id=1002, stmt_id_map_.size()=8, stmt_info_map_.size()=8, op.get_used_size()=35631, get_mem_high()=144955080, expired_stmt_ids.count()=0, closed_stmt_ids.count()=8) [2024-08-16 10:20:09.286585] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AB6-0-0] [lt=31] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:09.286643] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AB6-0-0] [lt=40] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774809283134000, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774809283134000, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:09.301150] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=14] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:20:09.301177] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=24] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=1, size_used=5330, mem_used=39515136) [2024-08-16 10:20:09.301812] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF269-0-0] [lt=11][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509945, log_task={header:{begin_lsn:{lsn:18544234684}, end_lsn:{lsn:18544234806}, log_id:53509945, min_scn:{val:1723774808890582003, v:0}, max_scn:{val:1723774808890582003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544234562}, prev_proposal_id:7, committed_end_lsn:{lsn:18544234684}, data_checksum:1913811101, accum_checksum:4088572091, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774809201735, freeze_ts:1723774809201735, submit_ts:1723774809201744, flushed_ts:1723774809202476, gen_to_freeze cost time:0, gen_to_submit cost time:9, submit_to_flush cost time:732}, fs_cb_begin_ts=1723774809301799, log_life_time=100064) [2024-08-16 10:20:09.302370] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=23322934, this={dio_aligned_buf:{buf_write_offset:432, buf_padding_size:3664, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:40624804, truncate_used_ts:10122096}, log_block_size:67108864, dir_fd:156, io_fd:95}, aligned_buf_len=4096, aligned_buf="LH", aligned_block_offset=23322624, buf_len=122, total_write_size_=14955, total_write_size_after_dio_=434176, ob_pwrite_used_ts=64090, count=102, each_pwrite_cost=628) [2024-08-16 10:20:09.302470] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=29] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=44, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:20:09.306922] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.306946] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:09.306962] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.306976] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.306995] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.307189] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD869C-0-0] [lt=80] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:09.307711] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=535, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:09.307733] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=20] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:09.307757] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=15] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=31, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:09.307838] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=37, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:09.308174] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA744B6-0-0] [lt=18] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:09.308195] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA744B6-0-0] [lt=20] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1011, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:09.347036] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [2749][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=18] run rewrite rule refresh task(rule_mgr_->tenant_id_=1001) [2024-08-16 10:20:09.351186] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:09.351300] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:09.351321] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:09.351329] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:09.352521] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=32][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:09.358177] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.358197] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:09.358207] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.358217] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.358230] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.375997] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:09.376286] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash time detail, (compute_wash_size_time=117, refresh_score_time=93, wash_time=16) [2024-08-16 10:20:09.377961] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=18] ====== traversal_flush timer task ====== [2024-08-16 10:20:09.378096] INFO [MDS] flush (mds_table_mgr.cpp:195) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=24] [FREEZE]no need do flush cause min_rec_scn is larger than freezing scn(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1079}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:32, ordered_tablets:{{tablet_id:{id:208204}, rec_scn:{val:1723723445809886004, v:0}}, {tablet_id:{id:1152921504606884836}, rec_scn:{val:1723723445809886006, v:0}}, {tablet_id:{id:1152921504606884831}, rec_scn:{val:1723723445809886007, v:0}}, {tablet_id:{id:208210}, rec_scn:{val:1723723445809886008, v:0}}, {tablet_id:{id:1152921504606884610}, rec_scn:{val:1723723445809886009, v:0}}, {tablet_id:{id:1152921504606895583}, rec_scn:{val:1723723445809886011, v:0}}, {tablet_id:{id:1152921504606884840}, rec_scn:{val:1723723445809886012, v:0}}, {tablet_id:{id:1152921504606884615}, rec_scn:{val:1723723445809886013, v:0}}, {tablet_id:{id:1152921504606903842}, rec_scn:{val:1723723446311099001, v:0}}, {tablet_id:{id:208206}, rec_scn:{val:1723723446311099002, v:0}}, {tablet_id:{id:208211}, rec_scn:{val:1723723446311099003, v:0}}, {tablet_id:{id:1152921504606884595}, rec_scn:{val:1723723446311099005, v:0}}, {tablet_id:{id:219872}, rec_scn:{val:1723723446311099006, v:0}}, {tablet_id:{id:208218}, rec_scn:{val:1723723446311099007, v:0}}, {tablet_id:{id:1152921504606884825}, rec_scn:{val:1723723446311099008, v:0}}, {tablet_id:{id:208208}, rec_scn:{val:1723723446311099010, v:0}}, {tablet_id:{id:1152921504606884606}, rec_scn:{val:1723723446311099011, v:0}}, {tablet_id:{id:1152921504606884588}, rec_scn:{val:1723723446311099012, v:0}}, {tablet_id:{id:1152921504606884580}, rec_scn:{val:1723723446311099013, v:0}}, {tablet_id:{id:1152921504606895579}, rec_scn:{val:1723723446311099015, v:0}}, {tablet_id:{id:1152921504606884601}, rec_scn:{val:1723723446311099016, v:0}}, {tablet_id:{id:1152921504606884609}, rec_scn:{val:1723723446311099017, v:0}}, {tablet_id:{id:1152921504606884618}, rec_scn:{val:1723723446311099018, v:0}}, {tablet_id:{id:1152921504606884839}, rec_scn:{val:1723723446311099019, v:0}}, {tablet_id:{id:1152921504606884599}, rec_scn:{val:1723723446311099021, v:0}}, {tablet_id:{id:1152921504606884616}, rec_scn:{val:1723723446311099022, v:0}}, {tablet_id:{id:1152921504606895582}, rec_scn:{val:1723723446812238001, v:0}}, {tablet_id:{id:1152921504606884581}, rec_scn:{val:1723723446812238002, v:0}}, {tablet_id:{id:1152921504606884590}, rec_scn:{val:1723723446812238003, v:0}}, {tablet_id:{id:1152921504606884827}, rec_scn:{val:1723723446812238005, v:0}}, {tablet_id:{id:1152921504606914006}, rec_scn:{val:1723723446812238006, v:0}}, {tablet_id:{id:214096}, rec_scn:{val:1723723446812238007, v:0}}, }, max_consequent_callbacked_scn={val:1723774808890582003, v:0}, *this={this:0x7fa6fa246510, is_inited:true, freezing_scn:{val:1721997282211836001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774805887235002, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:09.379657] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=93, clean_time=0, clean_start_pos=1195366, clean_num=62914) [2024-08-16 10:20:09.379766] INFO [MDS] flush (mds_table_mgr.cpp:195) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=142] [FREEZE]no need do flush cause min_rec_scn is larger than freezing scn(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1260}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:32, ordered_tablets:{{tablet_id:{id:1152921504606868225}, rec_scn:{val:1723723302639709002, v:0}}, {tablet_id:{id:1152921504606912559}, rec_scn:{val:1723723302639709003, v:0}}, {tablet_id:{id:1152921504606871965}, rec_scn:{val:1723723302639709005, v:0}}, {tablet_id:{id:212886}, rec_scn:{val:1723723302639709006, v:0}}, {tablet_id:{id:1152921504606858436}, rec_scn:{val:1723723302639709007, v:0}}, {tablet_id:{id:212255}, rec_scn:{val:1723723302639709009, v:0}}, {tablet_id:{id:1152921504606876671}, rec_scn:{val:1723723302639709010, v:0}}, {tablet_id:{id:1152921504606910457}, rec_scn:{val:1723723302639709012, v:0}}, {tablet_id:{id:214006}, rec_scn:{val:1723723302639709013, v:0}}, {tablet_id:{id:1152921504606912491}, rec_scn:{val:1723723302639709014, v:0}}, {tablet_id:{id:1152921504606890057}, rec_scn:{val:1723723303141598001, v:0}}, {tablet_id:{id:1152921504606848015}, rec_scn:{val:1723723303141598002, v:0}}, {tablet_id:{id:1152921504606847616}, rec_scn:{val:1723723303141598004, v:0}}, {tablet_id:{id:1152921504606913427}, rec_scn:{val:1723723303141598005, v:0}}, {tablet_id:{id:1152921504606915480}, rec_scn:{val:1723723303141598006, v:0}}, {tablet_id:{id:1152921504606889072}, rec_scn:{val:1723723303141598008, v:0}}, {tablet_id:{id:1152921504606915220}, rec_scn:{val:1723723303141598009, v:0}}, {tablet_id:{id:1152921504606866874}, rec_scn:{val:1723723303141598010, v:0}}, {tablet_id:{id:200529}, rec_scn:{val:1723723303141598012, v:0}}, {tablet_id:{id:1152921504606870508}, rec_scn:{val:1723723303141598013, v:0}}, {tablet_id:{id:1152921504606903696}, rec_scn:{val:1723723303141598015, v:0}}, {tablet_id:{id:1152921504606896782}, rec_scn:{val:1723723303647138001, v:0}}, {tablet_id:{id:1152921504606898434}, rec_scn:{val:1723723303647138002, v:0}}, {tablet_id:{id:1152921504606899144}, rec_scn:{val:1723723303647138004, v:0}}, {tablet_id:{id:1152921504606908791}, rec_scn:{val:1723723303829693001, v:0}}, {tablet_id:{id:1152921504606871378}, rec_scn:{val:1723723303829693002, v:0}}, {tablet_id:{id:1152921504606870897}, rec_scn:{val:1723723303829693003, v:0}}, {tablet_id:{id:1152921504606877002}, rec_scn:{val:1723723303829693004, v:0}}, {tablet_id:{id:1152921504606886744}, rec_scn:{val:1723723303829693006, v:0}}, {tablet_id:{id:1152921504606875352}, rec_scn:{val:1723723303829693007, v:0}}, {tablet_id:{id:204192}, rec_scn:{val:1723723303829693009, v:0}}, {tablet_id:{id:1152921504606897887}, rec_scn:{val:1723723304142080001, v:0}}, }, max_consequent_callbacked_scn={val:1723774808931219003, v:0}, *this={this:0x7fa6bd53e510, is_inited:true, freezing_scn:{val:1723690100831412000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1260}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202608224000, v:0}, clog_base_lsn:{lsn:8723619840}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774805887235002, v:0}, tablet_change_checkpoint_scn:{val:1723719123293042000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723639313794159001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1260, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:09.380347] INFO [MDS] flush (mds_table_mgr.cpp:172) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=110] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1261}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x7fa6fe204510, is_inited:true, freezing_scn:{val:1723752041922970000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1261}, ls_create_status:1, clog_checkpoint_scn:{val:1723752188579788000, v:0}, clog_base_lsn:{lsn:32545812480}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774805887235002, v:0}, tablet_change_checkpoint_scn:{val:1723719128263216001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723617733349446001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:4, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1261}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1261, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:09.381983] INFO [MDS] flush (mds_table_mgr.cpp:195) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=48] [FREEZE]no need do flush cause min_rec_scn is larger than freezing scn(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1262}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:32, ordered_tablets:{{tablet_id:{id:209134}, rec_scn:{val:1723723207973113002, v:0}}, {tablet_id:{id:1152921504606869956}, rec_scn:{val:1723723207973113003, v:0}}, {tablet_id:{id:1152921504606871991}, rec_scn:{val:1723723207973113005, v:0}}, {tablet_id:{id:1152921504606868203}, rec_scn:{val:1723723207973113006, v:0}}, {tablet_id:{id:1152921504606857189}, rec_scn:{val:1723723207973113007, v:0}}, {tablet_id:{id:1152921504606903903}, rec_scn:{val:1723723207973113009, v:0}}, {tablet_id:{id:205824}, rec_scn:{val:1723723207973113010, v:0}}, {tablet_id:{id:207632}, rec_scn:{val:1723723207973113011, v:0}}, {tablet_id:{id:1152921504606888997}, rec_scn:{val:1723723207973113013, v:0}}, {tablet_id:{id:214372}, rec_scn:{val:1723723207973113014, v:0}}, {tablet_id:{id:1152921504606915317}, rec_scn:{val:1723723207973113015, v:0}}, {tablet_id:{id:1152921504606875134}, rec_scn:{val:1723723207973113016, v:0}}, {tablet_id:{id:1152921504606876196}, rec_scn:{val:1723723208473765001, v:0}}, {tablet_id:{id:221202}, rec_scn:{val:1723723208473765002, v:0}}, {tablet_id:{id:1152921504606888619}, rec_scn:{val:1723723208473765003, v:0}}, {tablet_id:{id:1152921504606898978}, rec_scn:{val:1723723208473765005, v:0}}, {tablet_id:{id:1152921504606897298}, rec_scn:{val:1723723208473765006, v:0}}, {tablet_id:{id:1152921504606918529}, rec_scn:{val:1723723208473765007, v:0}}, {tablet_id:{id:1152921504606872365}, rec_scn:{val:1723723208473765009, v:0}}, {tablet_id:{id:1152921504606898136}, rec_scn:{val:1723723208473765010, v:0}}, {tablet_id:{id:210672}, rec_scn:{val:1723723208473765011, v:0}}, {tablet_id:{id:209037}, rec_scn:{val:1723723208473765013, v:0}}, {tablet_id:{id:1152921504606858115}, rec_scn:{val:1723723208473765014, v:0}}, {tablet_id:{id:1152921504606896677}, rec_scn:{val:1723723208473765015, v:0}}, {tablet_id:{id:1152921504606896744}, rec_scn:{val:1723723208473765016, v:0}}, {tablet_id:{id:218140}, rec_scn:{val:1723723208473765018, v:0}}, {tablet_id:{id:1152921504606856887}, rec_scn:{val:1723723208473765019, v:0}}, {tablet_id:{id:1152921504606856837}, rec_scn:{val:1723723208473765020, v:0}}, {tablet_id:{id:1152921504606867226}, rec_scn:{val:1723723208973569001, v:0}}, {tablet_id:{id:1152921504606912513}, rec_scn:{val:1723723208973569002, v:0}}, {tablet_id:{id:1152921504606856874}, rec_scn:{val:1723723208973569003, v:0}}, {tablet_id:{id:1152921504606896355}, rec_scn:{val:1723723208973569004, v:0}}, }, max_consequent_callbacked_scn={val:1723774809298179000, v:0}, *this={this:0x7fa6be4fc510, is_inited:true, freezing_scn:{val:1723656976361279000, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774805887235002, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-08-16 10:20:09.382112] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:336) [2559][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=123] succeed to traversal_flush(ret=0, ls_cnt=4) [2024-08-16 10:20:09.385031] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=21] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:09.402136] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF26F-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1623, avg_group_log_size=147) [2024-08-16 10:20:09.402188] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF26F-0-0] [lt=31][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509946, log_task={header:{begin_lsn:{lsn:18544234806}, end_lsn:{lsn:18544234928}, log_id:53509946, min_scn:{val:1723774808890582004, v:0}, max_scn:{val:1723774808890582004, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544234684}, prev_proposal_id:7, committed_end_lsn:{lsn:18544234806}, data_checksum:1913811101, accum_checksum:746915946, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774809301779, freeze_ts:1723774809301780, submit_ts:1723774809301783, flushed_ts:1723774809302433, gen_to_freeze cost time:1, gen_to_submit cost time:4, submit_to_flush cost time:650}, fs_cb_begin_ts=1723774809402174, log_life_time=100395) [2024-08-16 10:20:09.406992] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.407013] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:09.407024] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.407037] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.407058] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.426317] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=13] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:09.426317] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=23] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:09.443422] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=35] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=1) [2024-08-16 10:20:09.445669] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE88-0-0] [lt=19] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:09.445716] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE88-0-0] [lt=29] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774808909736004, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774808909736004, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:09.446425] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=26] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574656, offset=42151281}, end_cursor:ObLogCursor{file_id=65, log_id=18574657, offset=42151522}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751754050, len:655, log_data_len:241, seq:18574656, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:09.447041] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=43] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574657, offset=42151522}, end_cursor:ObLogCursor{file_id=65, log_id=18574658, offset=42151771}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75171c050, len:414, log_data_len:249, seq:18574657, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:09.448571] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCAB-0-0] [lt=19] push gts task success(*task={:""}) [2024-08-16 10:20:09.451251] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] tx gc loop thread is running(MTL_ID()=1002) [2024-08-16 10:20:09.452261] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1079}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202639465000, v:0}, clog_base_lsn:{lsn:18453811200}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774806764624003, v:0}, tablet_change_checkpoint_scn:{val:1723719117999212001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:2, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1079}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1079, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:20:09.452341] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=76] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:09.452598] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:09.453293] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1260}, ls_create_status:1, clog_checkpoint_scn:{val:1723723202608224000, v:0}, clog_base_lsn:{lsn:8723619840}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774806764624003, v:0}, tablet_change_checkpoint_scn:{val:1723719123293042000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723639313794159001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:1, proposal_id:29, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1260}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1260, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:20:09.453415] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=47] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:09.453944] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=596, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:09.453977] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=32, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:09.454021] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=14, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:09.454334] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1261}, ls_create_status:1, clog_checkpoint_scn:{val:1723752188579788000, v:0}, clog_base_lsn:{lsn:32545812480}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774805887235002, v:0}, tablet_change_checkpoint_scn:{val:1723719128263216001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723617733349446001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:4, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1261}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1261, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:20:09.454385] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=49] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:09.454541] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F736CF-0-0] [lt=31] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:09.454559] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F736CF-0-0] [lt=17] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=1298, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:09.454700] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCAB-0-0] [lt=18] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:09.455283] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1262}, ls_create_status:1, clog_checkpoint_scn:{val:1723723207973113002, v:0}, clog_base_lsn:{lsn:8253886464}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1723774805887235002, v:0}, tablet_change_checkpoint_scn:{val:1723719133124742002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1723646524294762001, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:1, log_handler:{role:2, proposal_id:1, palf_env_:0x7fa7105f4030, is_in_stop_state_:false, is_inited_:true, id_:1262}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1262, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-08-16 10:20:09.455327] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=42] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:09.456787] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=18] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574658, offset=42151771}, end_cursor:ObLogCursor{file_id=65, log_id=18574659, offset=42152020}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751748050, len:4261, log_data_len:249, seq:18574658, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:09.457180] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=38] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574659, offset=42152020}, end_cursor:ObLogCursor{file_id=65, log_id=18574660, offset=42152269}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75174c050, len:4012, log_data_len:249, seq:18574659, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:09.458248] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.458276] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:20:09.458291] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.458302] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.458324] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.492010] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7E2-0-0] [lt=25] [RS_MGR] force set rs list(master_rs="192.168.0.126:2882") [2024-08-16 10:20:09.492040] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [21121][T1_L0_G0][T1][YB42C0A8007E-00061EFF879CB7E2-0-0] [lt=23] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"192.168.0.128:2882", server_id:16, rs_addr:"192.168.0.126:2882", rs_server_status:1, epoch_id:11}, hb_response={zone:"zone2", server:"192.168.0.128:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1720514727760575, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=31) [2024-08-16 10:20:09.507091] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.507112] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:09.507123] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.507132] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.507146] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.507299] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD86A0-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:09.510414] INFO [CLOG] do_replay_task_ (ob_log_replay_service.cpp:901) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=13] wait for barrier(ret=-4023, replay_status={ls_id_:{id:1261}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:7, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774809510258, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:344950, ret_code_:0}}, next_to_submit_lsn_:{lsn:32601574498}, next_to_submit_scn_:{val:1723774809410647001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516552080113001, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd805000, next_round_pread_size:3678208, curr_read_pos:56, curr_read_buf_start_pos:0, curr_read_buf_end_pos:112, log_storage_:{IteratorStorage:{start_lsn:{lsn:32601574498}, end_lsn:{lsn:32601574610}, read_buf:{buf_len_:3682304, buf_:0x7fa6cd805000}, block_size:67104768, log_storage_:0x7fa6fe299470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:56, prev_entry_scn:{val:1723774809410647001, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:24, scn_:{val:1723774809410647001, v:0}, data_checksum:2297530913, flag:0}}, init_mode_version:0, accumulate_checksum:2142900629, curr_entry_is_padding:0, padding_entry_size:56, padding_entry_scn:{val:1723774809410647001, v:0}}}}}, replay_task={ls_id_:{id:1261}, log_type:17, log_type:"RESERVED_SNAPSHOT", lsn_:{lsn:32601574554}, scn_:{val:1723774809410647001, v:0}, is_pre_barrier_:true, is_post_barrier_:false, read_log_size_:24, replay_hint_:1723774809508402, first_handle_ts_:1723774809510412, replay_cost_:-1, retry_cost_:-1, read_log_buf_:0x7fa5a418d1f0, decompression_buf_:null, has_decompressed_:false, decompressed_log_size_:0, base_header_len_:16}) [2024-08-16 10:20:09.510463] INFO [CLOG] push_log_replay_task (ob_replay_status.cpp:1140) [2587][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=15] submit pre barrier log success(log_type=17, ls_id={id:1261}, lsn={lsn:32601574554}, scn={val:1723774809410647001, v:0}, is_pre_barrier=true, is_post_barrier=false, log_size=24) [2024-08-16 10:20:09.510611] INFO [STORAGE] inner_update_reserved_snapshot (ob_ls_reserved_snapshot_mgr.cpp:288) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=31] success to update reserved snapshot(ret=0, ls_id={id:1261}, min_reserved_snapshot_=1723773000638406000) [2024-08-16 10:20:09.510635] INFO [CLOG] do_replay_task_ (ob_log_replay_service.cpp:919) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=22] pre barrier log replay succ(replay_task={ls_id_:{id:1261}, log_type:17, log_type:"RESERVED_SNAPSHOT", lsn_:{lsn:32601574554}, scn_:{val:1723774809410647001, v:0}, is_pre_barrier_:true, is_post_barrier_:false, read_log_size_:24, replay_hint_:1723774809508402, first_handle_ts_:1723774809510594, replay_cost_:39, retry_cost_:-1, read_log_buf_:0x7fa5a418d200, decompression_buf_:null, has_decompressed_:false, decompressed_log_size_:0, base_header_len_:16}, replay_status={ls_id_:{id:1261}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774809510258, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:344950, ret_code_:0}}, next_to_submit_lsn_:{lsn:32601574610}, next_to_submit_scn_:{val:1723774809410647002, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516552080113001, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd805000, next_round_pread_size:3678208, curr_read_pos:112, curr_read_buf_start_pos:0, curr_read_buf_end_pos:112, log_storage_:{IteratorStorage:{start_lsn:{lsn:32601574498}, end_lsn:{lsn:32601574610}, read_buf:{buf_len_:3682304, buf_:0x7fa6cd805000}, block_size:67104768, log_storage_:0x7fa6fe299470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774809410647001, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:24, scn_:{val:1723774809410647001, v:0}, data_checksum:2297530913, flag:0}}, init_mode_version:0, accumulate_checksum:2142900629, curr_entry_is_padding:0, padding_entry_size:56, padding_entry_scn:{val:1723774809410647001, v:0}}}}}, replay_queue_idx=18, ret=0) [2024-08-16 10:20:09.527057] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=35] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7057, mem_used=12478464) [2024-08-16 10:20:09.534511] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=40] start do ls ha handler(ls_id_array_=[{id:1261}, {id:1260}, {id:1079}, {id:1262}]) [2024-08-16 10:20:09.542141] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=11] trace max work area(auto_calc=true, tenant_max_memory_limit=1073741824, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=53687091, max_wa_memory_size=53687091, tmp_max_wa_memory_size=53687091, pre_mem_target=53687091, remain_memory_size=53687091, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=332681216, washable_size=-2, max_workarea_memory_size=53687091, max_tenant_memory_size=741060608, wash_ratio=6, tenant_id=1001) [2024-08-16 10:20:09.542240] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=56] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=6710886, calc_global_bound_size=1048576000000, mem_target_=53687091, wa_max_memory_size=53687091, profile_cnt_=0, total_memory_size=0, max_wa_size=53687091, avg_bound_size=6710886, max_bound_size=6710886, min_bound_size_=1048576) [2024-08-16 10:20:09.542266] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=23] timer to calc global bound size(ret=0, global_bound_size_=6710886, manual_calc_cnt_=0, drift_size_=0, pre_drift_size=0, wa_max_memory_size=53687091, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1001, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=6710886, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=53687091, auto_calc=true, sql_mem_callback_.get_total_dump_size()=0) [2024-08-16 10:20:09.542311] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=11] trace max work area(auto_calc=true, tenant_max_memory_limit=6442450944, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=3865470566, max_wa_memory_size=3865470566, tmp_max_wa_memory_size=3865470566, pre_mem_target=3865470566, remain_memory_size=3865470566, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=2461601792, washable_size=-2, max_workarea_memory_size=3865470566, max_tenant_memory_size=3980849152, wash_ratio=6, tenant_id=1002) [2024-08-16 10:20:09.542367] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=20] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=483183820, calc_global_bound_size=1048576000000, mem_target_=3865470566, wa_max_memory_size=3865470566, profile_cnt_=0, total_memory_size=0, max_wa_size=3865470566, avg_bound_size=483183820, max_bound_size=483183820, min_bound_size_=1048576) [2024-08-16 10:20:09.542385] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [31390][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=17] timer to calc global bound size(ret=0, global_bound_size_=483183820, manual_calc_cnt_=19637, drift_size_=0, pre_drift_size=0, wa_max_memory_size=3865470566, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1002, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=483183820, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=3865470566, auto_calc=true, sql_mem_callback_.get_total_dump_size()=1112786688) [2024-08-16 10:20:09.545921] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [31638][T1_Occam][T1][YB42C0A80080-00061CCCBF6D3E12-0-0] [lt=6] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:09.551336] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:09.551439] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:09.551457] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:09.551464] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:09.552688] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:09.558342] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.558362] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:09.558372] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.558382] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.558397] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.559834] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B6B-0-0] [lt=23] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=22, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:09.580136] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=26] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:09.580428] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=36] Wash time detail, (compute_wash_size_time=157, refresh_score_time=79, wash_time=16) [2024-08-16 10:20:09.583540] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=74, clean_time=0, clean_start_pos=1258280, clean_num=62914) [2024-08-16 10:20:09.584732] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [31651][T1_PalfGC][T1][Y0-0000000000000000-0-0] [lt=27] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}) [2024-08-16 10:20:09.585067] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=10] ====== check clog disk timer task ====== [2024-08-16 10:20:09.585101] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=31] get_disk_usage(ret=0, capacity(MB):=2150, used(MB):=0) [2024-08-16 10:20:09.585114] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2784][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=8] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=676457349, need_update_checkpoint_scn=false) [2024-08-16 10:20:09.585429] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 387.00) [2024-08-16 10:20:09.585455] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=20] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:09.585529] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=32] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 5120.00, iops: 3.99, rt: 450.75) [2024-08-16 10:20:09.585546] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=14] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:09.585584] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=19] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585600] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585608] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585614] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585625] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585630] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585634] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585647] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=12] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585654] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585661] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585679] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585688] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585703] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=13] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585710] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585715] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.585721] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:09.586226] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=20] ====== check clog disk timer task ====== [2024-08-16 10:20:09.586245] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=17] get_disk_usage(ret=0, capacity(MB):=0, used(MB):=0) [2024-08-16 10:20:09.586261] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [31773][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=10] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=0, need_update_checkpoint_scn=false) [2024-08-16 10:20:09.591767] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=13][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:09.594145] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=22] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:09.594240] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=21][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:09.594269] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=27][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:09.594305] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=34] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:09.601941] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF27B-0-0] [lt=24] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1623) [2024-08-16 10:20:09.601988] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF27B-0-0] [lt=21] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=10, avg_log_submit_to_flush_time=714, avg_log_submit_to_slide_time=90980) [2024-08-16 10:20:09.602009] INFO [PALF] read (log_cache.cpp:100) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=70] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1623, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:09.602652] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=40] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=33, stat_interval=1000000, avg (count/cost)=31, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:09.607141] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.607163] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:09.607174] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.607183] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.607205] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.607978] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=14] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:09.610492] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [31774][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=9] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:20:09.610521] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:230) [31774][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=22] [tabletchange] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12639) [2024-08-16 10:20:09.618647] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=24] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60751, mem_used=137263104) [2024-08-16 10:20:09.643525] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=23] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=0) [2024-08-16 10:20:09.651419] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:09.651548] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:09.651564] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:09.651571] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:09.652766] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=21][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:09.658431] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.658453] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:09.658462] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.658471] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.658497] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.661159] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=29] [PALF STAT WRITE LOG COST TIME](cur_stat_count=22, stat_interval=1000000, avg (count/cost)=529, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:09.661306] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=25] [PALF STAT FLUSH CB COST TIME](cur_stat_count=22, stat_interval=1000000, avg (count/cost)=35, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:09.693658] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=24] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:09.702136] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF281-0-0] [lt=18][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509949, log_task={header:{begin_lsn:{lsn:18544235172}, end_lsn:{lsn:18544235294}, log_id:53509949, min_scn:{val:1723774809391280003, v:0}, max_scn:{val:1723774809391280003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544235050}, prev_proposal_id:7, committed_end_lsn:{lsn:18544235172}, data_checksum:1913811101, accum_checksum:3630354593, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774809601919, freeze_ts:1723774809601919, submit_ts:1723774809601926, flushed_ts:1723774809602673, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:747}, fs_cb_begin_ts=1723774809702123, log_life_time=100204) [2024-08-16 10:20:09.707217] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.707242] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:09.707259] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.707274] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.707303] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.751504] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:09.751602] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=29] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=45, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:09.751636] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:09.751655] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:09.751665] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:09.751678] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:09.752777] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F736D8-0-0] [lt=61] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=5, avg_log_gen_to_submit_time=63, avg_log_submit_to_flush_time=670, avg_log_submit_to_slide_time=1141) [2024-08-16 10:20:09.752841] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:09.758499] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.758520] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:09.758529] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.758538] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.758552] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.759844] INFO [STORAGE.TRANS] rollback_tx (ob_tx_api.cpp:339) [2568][T1002_ArcSrv][T1001][YB42C0A80080-00061CCC9251D0C5-0-0] [lt=80] rollback tx(ret=0, *this={is_inited_:true, tenant_id_:1001, this:0x7fa6e0004030}, tx={this:0x7fa704c0f4a0, tx_id:{txid:283711691}, state:7, addr:"192.168.0.128:2882", tenant_id:1001, session_id:3225596091, assoc_session_id:3225596091, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:1723774809752252, op_sn:3, alloc_ts:1723774809753303, active_ts:1723774809753303, commit_ts:-1, finish_ts:1723774809759665, timeout_us:29999985, lock_timeout_us:-1, expire_ts:1723774839753288, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:1713182723, cluster_version:17180000768, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:-6002, commit_expire_ts:0, commit_task_.is_registered():false, ref:2}) [2024-08-16 10:20:09.760116] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B6F-0-0] [lt=22] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=22, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:09.760141] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B6F-0-0] [lt=23] [PALF STAT LOG LIFE TIME](cur_stat_count=22, stat_interval=1000000, avg (count/cost)=5457, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:09.760152] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B6F-0-0] [lt=9][errcode=-4389] log_task life cost too much time(palf_id=1262, self="192.168.0.128:2882", log_id=34459453, log_task={header:{begin_lsn:{lsn:8394860770}, end_lsn:{lsn:8394860892}, log_id:34459453, min_scn:{val:1723774809298179003, v:0}, max_scn:{val:1723774809298179003, v:0}, data_len:66, proposal_id:2, prev_lsn:{lsn:8394860648}, prev_proposal_id:2, committed_end_lsn:{lsn:8394860770}, data_checksum:2071192402, accum_checksum:2798789696, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774809659926, freeze_ts:1723774809659927, submit_ts:1723774809659931, flushed_ts:1723774809661227, gen_to_freeze cost time:1, gen_to_submit cost time:5, submit_to_flush cost time:1296}, fs_cb_begin_ts=1723774809760101, log_life_time=100175) [2024-08-16 10:20:09.766879] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=23] LogLoopThread round_cost_time(us)(round_cost_time=20) [2024-08-16 10:20:09.772496] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=27] ====== tenant freeze timer task ====== [2024-08-16 10:20:09.772526] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [31673][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=19][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:09.782309] INFO [COMMON] generate_mod_stat_task (memory_dump.cpp:218) [31418][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=17] task info(*task={type_:2, dump_all_:false, p_context_:null, slot_idx_:0, dump_tenant_ctx_:false, tenant_id_:0, ctx_id_:0, p_chunk_:null}) [2024-08-16 10:20:09.782340] INFO [COMMON] handle (memory_dump.cpp:498) [31418][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=29] handle dump task(task={type_:2, dump_all_:false, p_context_:null, slot_idx_:0, dump_tenant_ctx_:false, tenant_id_:0, ctx_id_:0, p_chunk_:null}) [2024-08-16 10:20:09.782472] INFO [COMMON] update_check_range (ob_sql_mem_leak_checker.cpp:62) [31418][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=10] update_check_range(min_check_version=324703, max_check_version=324704, global_version=324705) [2024-08-16 10:20:09.783976] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=23] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:09.784369] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=34] Wash time detail, (compute_wash_size_time=127, refresh_score_time=142, wash_time=31) [2024-08-16 10:20:09.787644] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=16] Cache wash clean map node details(ret=0, clean_node_count=111, clean_time=0, clean_start_pos=1321194, clean_num=62914) [2024-08-16 10:20:09.802396] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF287-0-0] [lt=44] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:09.802473] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF287-0-0] [lt=34] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=11, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:09.802494] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF287-0-0] [lt=20] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=91027, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:09.807291] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=26] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.807317] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] handle task succ(id={id:1260}) [2024-08-16 10:20:09.807332] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.807342] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.807364] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.807989] INFO [SERVER] reload_tenant_task (ob_tenant_tablet_ttl_mgr.cpp:1297) [5766][T1002_TTLTablet][T1002][YB42C0A80080-00061CCC98B6C841-0-0] [lt=43] finish reload tenant task(local_tenant_task_={tenant_id:1002, task_id:-1, is_usr_trigger:false, need_check:false, is_dirty:false, ttl_continue:true, rsp_time:-1, state:4, is_reused:true}, tenant_task={gmt_create:0, gmt_modified:0, tenant_id:18446744073709551615, table_id:18446744073709551615, tablet_id:18446744073709551615, task_id:0, task_start_time:0, task_update_time:0, trigger_type:0, status:19, ttl_del_cnt:0, max_version_del_cnt:0, scan_cnt:0, row_key:"", ret_code:"OB_SUCCESS"}, is_leader=false) [2024-08-16 10:20:09.812475] INFO handle (memory_dump.cpp:626) [31418][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=20] statistics: tenant_cnt: 5, max_chunk_cnt: 524288 tenant_id ctx_id chunk_cnt label_cnt segv_cnt 1 0 202 181 0 1 5 22 5 0 1 7 1 2 0 1 8 54 1 0 1 12 8 3 0 1 13 5 1 0 1 14 1 1 0 1 20 1 1 0 500 0 113 256 0 500 7 13 4 0 500 8 55 1 0 500 9 2 1 0 500 10 6 2 0 500 16 3 1 0 500 17 5 7 0 500 22 55 58 0 500 23 27 11 0 508 0 7 13 0 508 8 13 1 0 1001 0 77 143 0 1001 5 23 5 0 1001 8 41 1 0 1001 12 1 1 0 1001 14 1 1 0 1002 0 482 237 0 1002 2 138 1 0 1002 4 2 1 0 1002 5 129 5 0 1002 8 68 1 0 1002 12 108 3 0 1002 13 21 1 0 1002 14 1 2 0 1002 15 1 1 0 1002 18 30 3 0 1002 20 20 1 0 cost_time: 30023 [2024-08-16 10:20:09.845240] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=27] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=2768216, replace_num=31457) [2024-08-16 10:20:09.845273] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=30] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=10) [2024-08-16 10:20:09.851587] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:09.851743] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=32] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:09.851769] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:09.851780] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:09.852419] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=30] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1211) [2024-08-16 10:20:09.852912] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:09.858591] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.858613] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:09.858625] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.858639] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=13] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.858662] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.862955] INFO [SERVER] try_reload_schema (ob_server_schema_updater.cpp:440) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=11] schedule fetch new schema task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:09.862977] INFO [SERVER] do_heartbeat_event (ob_heartbeat.cpp:268) [31844][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=21] try reload schema success(schema_version=1722933080671200, refresh_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_ret=0) [2024-08-16 10:20:09.862986] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:255) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E82-0-0] [lt=21] [REFRESH_SCHEMA] start to process schema refresh task(ret=0, ret="OB_SUCCESS", schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:09.863009] INFO [SERVER] process_refresh_task (ob_server_schema_updater.cpp:280) [31451][SerScheQueue0][T0][YB42C0A80080-00061CCC8D9E5E82-0-0] [lt=19] [REFRESH_SCHEMA] local schema info is newer, no need to refresh schema(ret=0, ret="OB_SUCCESS", local_schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}, schema_info={schema_version:1723771644364976, tenant_id:1002, sequence_id:13194139534868}) [2024-08-16 10:20:09.872237] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [2782][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=11] ====== checkpoint timer task ====== [2024-08-16 10:20:09.872262] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [2782][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=19] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:20:09.902438] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF28D-0-0] [lt=10][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509951, log_task={header:{begin_lsn:{lsn:18544235416}, end_lsn:{lsn:18544235538}, log_id:53509951, min_scn:{val:1723774809391280005, v:0}, max_scn:{val:1723774809391280005, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544235294}, prev_proposal_id:7, committed_end_lsn:{lsn:18544235416}, data_checksum:1913811101, accum_checksum:1482685115, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774809802383, freeze_ts:1723774809802383, submit_ts:1723774809802434, flushed_ts:1723774809803206, gen_to_freeze cost time:0, gen_to_submit cost time:51, submit_to_flush cost time:772}, fs_cb_begin_ts=1723774809902420, log_life_time=100037) [2024-08-16 10:20:09.903035] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=635, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:09.903058] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1342) [2024-08-16 10:20:09.907362] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.907385] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:09.907396] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.907407] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.907420] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.917433] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [2520][T1002_TenantMet][T1002][Y0-0000000000000000-0-0] [lt=45] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:17374, free_obj_hold_cnt:593, allocator used:71436792, allocator total:73453184}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:1994, free_obj_hold_cnt:6, allocator used:131182000, allocator total:131182000}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():63488, full allocator used:1600, full allocator total:129024}, tablets_mem=204764208, tablets_mem_limit=1288490180, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:7, allocator used:17024, allocator total:65408}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:106, free_obj_hold_cnt:5057, allocator used:10243392, allocator total:10596096}, pending_cnt=0, wait_gc_count=0, tablet count=19368) [2024-08-16 10:20:09.932937] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80081-00061EFF86EC941D-0-0] [lt=21][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:09.933819] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=33] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:09.951663] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:09.951779] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:09.951803] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=22] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:09.951816] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:09.952983] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:09.958334] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2417][T1002_Occam][T1002][YB42C0A80080-00061CCCBF7D3868-0-0] [lt=21] for each ls(succ_num=4, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:09.958656] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:09.958682] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:20:09.958693] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:09.958705] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:09.958719] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:09.962383] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=19] ====== check clog disk timer task ====== [2024-08-16 10:20:09.962415] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=23] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:09.962438] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [2560][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=10] cannot_recycle_log_size statistics(cannot_recycle_log_size=391698068, threshold=6088116142, need_update_checkpoint_scn=false) [2024-08-16 10:20:09.988089] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=24] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:09.988366] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=137, refresh_score_time=67, wash_time=36) [2024-08-16 10:20:09.991383] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=28] Cache wash clean map node details(ret=0, clean_node_count=85, clean_time=0, clean_start_pos=1384108, clean_num=62914) [2024-08-16 10:20:09.998526] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=8] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:09.998566] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=28] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:09.998714] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:09.998768] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=45][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:09.998784] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:09.998803] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=16][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:10.001116] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [31728][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=25] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-08-16 10:20:10.007437] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.007476] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=36] handle task succ(id={id:1260}) [2024-08-16 10:20:10.007487] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.007497] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.007513] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.013015] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=22] gc stale ls task succ [2024-08-16 10:20:10.013133] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=3] gc stale ls task succ [2024-08-16 10:20:10.013163] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=19] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:10.013178] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=10] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:10.013191] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=11] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:10.016885] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:10.016885] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=37] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:10.032872] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C33-0-0] [lt=45] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774810032839) [2024-08-16 10:20:10.045371] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=9] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=9) [2024-08-16 10:20:10.051740] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:10.051843] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:10.051865] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:10.051871] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:10.053055] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=35][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:10.058725] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.058746] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:10.058758] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.058775] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=15] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.058802] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.059888] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=25] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:10.065697] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:10.065726] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=28][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:10.066265] INFO [PALF] runTimerTask (block_gc_timer_task.cpp:101) [31651][T1_PalfGC][T1][Y0-0000000000000000-0-0] [lt=27] BlockGCTimerTask success(ret=0, cost_time_us=17, palf_env_impl_={IPalfEnvImpl:{IPalfEnvImpl:"Dummy"}, self:"192.168.0.128:2882", log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1", disk_options_wrapper:{disk_opts_for_stopping_writing:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):0, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}, log_alloc_mgr_:{flying_log_task:0, flying_meta_task:0}}) [2024-08-16 10:20:10.102502] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=23] PNIO [ratelimit] time: 1723774810102499, bytes: 1496438057772, bw: 0.155025 MB/s, add_ts: 1003401, add_bytes: 163108 [2024-08-16 10:20:10.102713] INFO [LIB] stat (utility.h:1197) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=23] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=75, stat_interval=1000000, avg (count/cost)=66, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:10.102739] INFO [LIB] stat (utility.h:1197) [2589][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=25] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=75, stat_interval=1000000, avg (count/cost)=20, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:10.105864] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [2783][T1001_Flush][T1001][Y0-0000000000000000-0-0] [lt=7] ====== traversal_flush timer task ====== [2024-08-16 10:20:10.105900] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:338) [2783][T1001_Flush][T1001][Y0-0000000000000000-0-0] [lt=29] no logstream(ret=0, ls_cnt=0) [2024-08-16 10:20:10.107529] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.107569] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=38] handle task succ(id={id:1260}) [2024-08-16 10:20:10.107581] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.107599] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.107613] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.107682] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD86AC-0-0] [lt=29] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:10.108741] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA744CF-0-0] [lt=21] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=7, avg_log_submit_to_flush_time=617, avg_log_submit_to_slide_time=1030) [2024-08-16 10:20:10.108762] INFO [PALF] read (log_cache.cpp:100) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=44] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1323, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:10.133097] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=40] PNIO [ratelimit] time: 1723774810133093, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007624, add_bytes: 0 [2024-08-16 10:20:10.133256] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [2659][T1001_Occam][T1001][YB42C0A80080-00061CCCBF8D3868-0-0] [lt=27] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-08-16 10:20:10.151822] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:10.151932] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1211, avg_group_log_size=121) [2024-08-16 10:20:10.151958] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1211) [2024-08-16 10:20:10.151974] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=13] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:10.151982] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:10.151987] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:10.152613] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1211) [2024-08-16 10:20:10.153150] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:120) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=39] tenant info is invalid(ret=-4076, query_ts=0, standby_scn={val:0, v:0}, this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"192.168.0.128:2882", switch_to_leader_ts:-1}) [2024-08-16 10:20:10.153194] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=41][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:10.158028] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7E4-0-0] [lt=131][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-16 10:20:10.158650] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=83] current clock is (clock_=0, max_seq_=0, mem_limit=1643328294, cur_mem_hold=289406976, attr_.tenant_id_=1002) [2024-08-16 10:20:10.158679] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=21] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1723774800153600, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1723774810157731, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-08-16 10:20:10.158711] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [31758][T1_LSMetaCh][T1][YB42C0A80080-00061CCCC758D7E4-0-0] [lt=20] fetch root ls success(local_cluster_id=1713182723, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.126:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:11, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1016, zone:"zone1", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932441752355, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932514131235, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.129:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1017, zone:"zone2", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932475191946, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1722932513390762, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.134:2882", sql_port:2881, role:2, member_list:[{server:"192.168.0.126:2882", timestamp:1722932441752355}, {server:"192.168.0.129:2882", timestamp:1722932475191946}, {server:"192.168.0.134:2882", timestamp:1722932495680366}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1018, zone:"zone3", paxos_replica_number:3, data_size:0, required_size:0, in_member_list:true, member_time_us:1722932495680366, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["192.168.0.126:2882", "192.168.0.129:2882", "192.168.0.134:2882"]) [2024-08-16 10:20:10.158803] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.158816] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=12] handle task succ(id={id:1260}) [2024-08-16 10:20:10.158823] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.158834] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.158856] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.160422] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B77-0-0] [lt=29] [PALF STAT GROUP LOG INFO](palf_id=1262, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=22, avg_log_batch_cnt=0, total_group_log_size=6574, avg_group_log_size=298) [2024-08-16 10:20:10.160444] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B77-0-0] [lt=20] [PALF STAT COMMITTED LOG SIZE](palf_id=1262, self="192.168.0.128:2882", committed size=6574) [2024-08-16 10:20:10.161026] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=28] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1262, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=6574) [2024-08-16 10:20:10.161464] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F736E6-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1262, self="192.168.0.128:2882", total_slide_log_cnt=22, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=5, avg_log_submit_to_flush_time=593, avg_log_submit_to_slide_time=5457) [2024-08-16 10:20:10.161484] INFO [PALF] read (log_cache.cpp:100) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=53] [PALF STAT HOT CACHE HIT RATE](palf_id=1262, read_size=6574, hit_cnt=22, read_cnt=22, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:10.163227] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007F-000616D421E91A10-0-0] [lt=24][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:10.167239] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2457][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=16] ====== tenant freeze timer task ====== [2024-08-16 10:20:10.184237] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2787][T1001_LSArchive][T1001][YB42C0A80080-00061CCC9321D0C5-0-0] [lt=19] gc stale ls task succ [2024-08-16 10:20:10.187743] INFO [CLOG.EXTLOG] resize_log_ext_handler_ (ob_cdc_service.cpp:632) [31748][T1_CdcSrv][T1][Y0-0000000000000000-0-0] [lt=22] finish to resize log external storage handler(current_ts=1723774810187731, tenant_max_cpu=2, valid_ls_count=0, other_ls_count=0, new_concurrency=0) [2024-08-16 10:20:10.191848] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:10.192127] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=25] Wash time detail, (compute_wash_size_time=147, refresh_score_time=66, wash_time=16) [2024-08-16 10:20:10.195388] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=16] Cache wash clean map node details(ret=0, clean_node_count=79, clean_time=0, clean_start_pos=1447022, clean_num=62914) [2024-08-16 10:20:10.203967] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [31841][T1_L0_G0][T1][YB42C0A80080-00061CCCC65F2E32-0-0] [lt=24][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=8, len=116, pos=86, bitmap=NULL) [2024-08-16 10:20:10.207621] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [2774][T1001_LCLSender][T1001][Y0-0000000000000000-0-0] [lt=45] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=1135, total_destructed_detector=1135, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x7fa704be42b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-08-16 10:20:10.207633] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.207649] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] handle task succ(id={id:1260}) [2024-08-16 10:20:10.207676] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.207692] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.207706] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.207769] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF29F-0-0] [lt=17][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509955, log_task={header:{begin_lsn:{lsn:18544236185}, end_lsn:{lsn:18544236307}, log_id:53509955, min_scn:{val:1723774809892173003, v:0}, max_scn:{val:1723774809892173003, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544235782}, prev_proposal_id:7, committed_end_lsn:{lsn:18544236185}, data_checksum:1913811101, accum_checksum:3492260523, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774810102541, freeze_ts:1723774810102541, submit_ts:1723774810102548, flushed_ts:1723774810103350, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:802}, fs_cb_begin_ts=1723774810207757, log_life_time=105216) [2024-08-16 10:20:10.236269] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31547][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=20] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:10.236407] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31549][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=17] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:10.236427] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31544][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:10.236669] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31542][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:10.236669] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [31543][BatchIO][T0][Y0-0000000000000000-0-0] [lt=9] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:10.236918] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [31546][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:10.237737] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31551][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:10.245481] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=30] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=8) [2024-08-16 10:20:10.251908] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:10.252050] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=19] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:10.252098] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=44] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:10.252109] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:10.253221] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=13][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:10.258893] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.258917] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=22] handle task succ(id={id:1260}) [2024-08-16 10:20:10.258928] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.258939] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.258951] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.267318] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=19] re_register_parent reason(palf_id=1262, self="192.168.0.128:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1723774806264242, last_first_register_time_us=1720517744275280, register_time_us=1723774806264242, curr_time_us=1723774810267316) [2024-08-16 10:20:10.267386] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=30] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1262, self="192.168.0.128:2882", reason=1, leader="192.168.0.133:2882", register_time_us=1723774810267350, last_submit_register_req_time_us=1723774810267350, register_parent_reason=1) [2024-08-16 10:20:10.267413] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=25] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1262, self="192.168.0.128:2882") [2024-08-16 10:20:10.267439] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=11] re_register_parent reason(palf_id=1261, self="192.168.0.128:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1723774806264349, last_first_register_time_us=1720517744277428, register_time_us=1723774806264349, curr_time_us=1723774810267438) [2024-08-16 10:20:10.267485] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=19] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1261, self="192.168.0.128:2882", reason=1, leader="192.168.0.125:2882", register_time_us=1723774810267459, last_submit_register_req_time_us=1723774810267459, register_parent_reason=1) [2024-08-16 10:20:10.267503] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [2435][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=17] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1261, self="192.168.0.128:2882") [2024-08-16 10:20:10.270794] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [31849][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=22] current monitor number(seq_=-1) [2024-08-16 10:20:10.287227] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [2566][T1002_ArcSender][T1002][YB42C0A80080-00061CCC9231D0C5-0-0] [lt=22][errcode=-4018] try free send task failed(ret=-4018) [2024-08-16 10:20:10.301308] INFO [SERVER] check_config_mem_limit (ob_eliminate_task.cpp:78) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=12] change config mem limit(config_mem_limit_=67108864, mem_limit=32212254, tenant_id=1) [2024-08-16 10:20:10.301341] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [31769][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=29] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=46137345, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=5341, mem_used=39515136) [2024-08-16 10:20:10.303516] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=27] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=41, this=0x7fa6fc6f35e0, extra_info="palf_id:1079") [2024-08-16 10:20:10.304590] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A80085-00061CCD1FB68AB7-0-0] [lt=26] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1262}}) [2024-08-16 10:20:10.304637] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A80085-00061CCD1FB68AB7-0-0] [lt=30] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774809769275004, v:0}, arg={tenant_id:1002, ls_id:{id:1262}}, result={tenant_id:1002, ls_id:{id:1262}, cur_readable_scn:{val:1723774809769275004, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:10.307713] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.307739] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=24] handle task succ(id={id:1260}) [2024-08-16 10:20:10.307750] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.307758] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.307778] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.307934] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD86AE-0-0] [lt=53] [PALF STAT COMMITTED LOG SIZE](palf_id=1261, self="192.168.0.128:2882", committed size=1770) [2024-08-16 10:20:10.308545] INFO [LIB] stat (utility.h:1197) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=564, this=0x7fa6fe29d3b0, extra_info="palf_id:1261") [2024-08-16 10:20:10.308577] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=30] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1261, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1770) [2024-08-16 10:20:10.308599] INFO [LIB] stat (utility.h:1214) [2431][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=30, this=0x7fa7105fbaf0, extra_info="") [2024-08-16 10:20:10.308681] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=33, this=0x7fa6fe29d5e0, extra_info="palf_id:1261") [2024-08-16 10:20:10.309122] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE712-0-0] [lt=19] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6fe2936a0, extra_info="palf_id:1261") [2024-08-16 10:20:10.309140] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE712-0-0] [lt=17] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=1065, this=0x7fa6fe2938d0, extra_info="palf_id:1261") [2024-08-16 10:20:10.312005] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=18] ====== tenant freeze timer task ====== [2024-08-16 10:20:10.312034] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [2694][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=17][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-08-16 10:20:10.352009] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:10.352134] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=23] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:10.352152] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=15] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:10.352163] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:10.353300] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:10.358956] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.358977] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:10.358987] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.358996] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.359010] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.371542] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [2512][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=80] get all snapshots(ret=0, snapshots.count()=192, snapshots=[{snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213191, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213192, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213193, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213194, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213195, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213196, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213197, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213198, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213199, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213200, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213201, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213202, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213203, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213204, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213205, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213206, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213207, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213208, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213209, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213210, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213211, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213212, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213213, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213214, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213215, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213216, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213217, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213218, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213219, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213220, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213221, comment:null}, {snapshot_type:1, snapshot_scn:{val:1720086189194443000, v:0}, schema_version:1720086182443672, tenant_id:1002, tablet_id:213222, comment:null}]) [2024-08-16 10:20:10.385094] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1506) [31513][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=18] skip inspect bad block(last_check_time=1723686634905356, last_macro_idx=-1) [2024-08-16 10:20:10.395817] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=23] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:10.396142] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=32] Wash time detail, (compute_wash_size_time=122, refresh_score_time=79, wash_time=26) [2024-08-16 10:20:10.399382] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Cache wash clean map node details(ret=0, clean_node_count=82, clean_time=0, clean_start_pos=1509936, clean_num=62914) [2024-08-16 10:20:10.400332] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [2562][T1002_TabletShe][T1002][Y0-0000000000000000-0-0] [lt=42] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:20:10.400367] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:99) [2562][T1002_TabletShe][T1002][Y0-0000000000000000-0-0] [lt=24] [emptytablet] succeed to change tablet to empty shell(ret=0, ret="OB_SUCCESS", ls_cnt=4, times=12268) [2024-08-16 10:20:10.402739] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF2AB-0-0] [lt=23] [PALF STAT GROUP LOG INFO](palf_id=1079, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1614, avg_group_log_size=146) [2024-08-16 10:20:10.402778] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF2AB-0-0] [lt=22][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509957, log_task={header:{begin_lsn:{lsn:18544236420}, end_lsn:{lsn:18544236542}, log_id:53509957, min_scn:{val:1723774809892173005, v:0}, max_scn:{val:1723774809892173005, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544236307}, prev_proposal_id:7, committed_end_lsn:{lsn:18544236420}, data_checksum:1913811101, accum_checksum:94976362, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774810302726, freeze_ts:1723774810302726, submit_ts:1723774810302737, flushed_ts:1723774810303479, gen_to_freeze cost time:0, gen_to_submit cost time:11, submit_to_flush cost time:742}, fs_cb_begin_ts=1723774810402767, log_life_time=100041) [2024-08-16 10:20:10.407786] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.407807] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:10.407818] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.407828] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.407842] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.426421] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [31786][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=18] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:10.426431] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2797][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=23] start do ls ha handler(ls_id_array_=[]) [2024-08-16 10:20:10.434402] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=22] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1, need_refresh=false, gts_local_cache={srr:[mts=1723774809933790], gts:1723774809933646000, latest_srr:[mts=1723774810434374]}) [2024-08-16 10:20:10.434444] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=28] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_refresh=false, gts_local_cache={srr:[mts=1723774809933843], gts:1723774809931967000, latest_srr:[mts=1723774810434432]}) [2024-08-16 10:20:10.434664] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=9] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_refresh=false, gts_local_cache={srr:[mts=1723774809934061], gts:1723774809933007000, latest_srr:[mts=1723774810434642]}) [2024-08-16 10:20:10.445600] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=23] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=7) [2024-08-16 10:20:10.452093] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:10.452218] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:10.452235] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:10.452243] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:10.453369] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=20][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:10.458686] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=22] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574660, offset=42152269}, end_cursor:ObLogCursor{file_id=65, log_id=18574661, offset=42152510}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751780050, len:3763, log_data_len:241, seq:18574660, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:10.459116] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=29] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574661, offset=42152510}, end_cursor:ObLogCursor{file_id=65, log_id=18574662, offset=42152759}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa7516fc050, len:3522, log_data_len:249, seq:18574661, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:10.459193] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.459213] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:10.459222] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.459232] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.459245] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.459256] INFO [ARCHIVE] do_thread_task_ (ob_archive_fetcher.cpp:312) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] ObArchiveFetcher is running(thread_index=1) [2024-08-16 10:20:10.459754] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [2809][T1001_L0_G0][T1001][YB42C0A8007D-00061CCCA753EE89-0-0] [lt=20] start get_ls_replayed_scn(arg={tenant_id:1002, ls_id:{id:1261}}) [2024-08-16 10:20:10.459826] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [2809][T1001_L0_G0][T1002][YB42C0A8007D-00061CCCA753EE89-0-0] [lt=29] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1723774810154993002, v:0}, arg={tenant_id:1002, ls_id:{id:1261}}, result={tenant_id:1002, ls_id:{id:1261}, cur_readable_scn:{val:1723774810154993002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-08-16 10:20:10.460153] INFO [STORAGE.TRANS] generate_new_version (ob_tenant_weak_read_server_version_mgr.cpp:120) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=22] [WRS] update tenant weak read server version(tenant_id=1001, server_version={version:{val:1723774805932890000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0, epoch_tstamp:1723774810460138}, version_delta=4527262) [2024-08-16 10:20:10.460565] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCAC-0-0] [lt=25] push gts task success(*task={:""}) [2024-08-16 10:20:10.460574] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B7D-0-0] [lt=32][errcode=-4389] log_task life cost too much time(palf_id=1262, self="192.168.0.128:2882", log_id=34459460, log_task={header:{begin_lsn:{lsn:8394861624}, end_lsn:{lsn:8394861746}, log_id:34459460, min_scn:{val:1723774810269649000, v:0}, max_scn:{val:1723774810269649000, v:0}, data_len:66, proposal_id:2, prev_lsn:{lsn:8394861502}, prev_proposal_id:2, committed_end_lsn:{lsn:8394861624}, data_checksum:2071192402, accum_checksum:3155696287, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774810360511, freeze_ts:1723774810360511, submit_ts:1723774810360516, flushed_ts:1723774810361570, gen_to_freeze cost time:0, gen_to_submit cost time:5, submit_to_flush cost time:1054}, fs_cb_begin_ts=1723774810460564, log_life_time=100053) [2024-08-16 10:20:10.480218] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [2767][T1001_LockWaitM][T1001][Y0-0000000000000000-0-0] [lt=25] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-08-16 10:20:10.480967] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [2443][T1002_TeRec][T1001][YB42C0A80080-00061CCCC807BCAC-0-0] [lt=18] rpc statistics(total_trans_msg_count=1, total_batch_msg_count=0) [2024-08-16 10:20:10.483505] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=22] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574662, offset=42152759}, end_cursor:ObLogCursor{file_id=65, log_id=18574663, offset=42153008}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa751784050, len:3273, log_data_len:249, seq:18574662, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:10.484138] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [2521][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=28] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=65, log_id=18574663, offset=42153008}, end_cursor:ObLogCursor{file_id=65, log_id=18574664, offset=42153257}, is_inited:true, is_local:false, buf_size:8192, buf:0x7fa75179c050, len:3024, log_data_len:249, seq:18574663, flush_finish:false, flush_ret:0}) [2024-08-16 10:20:10.507806] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=31] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1002, server_version={version:{val:1723774809769275004, v:0}, total_part_count:4, valid_inner_part_count:0, valid_user_part_count:4}, server_version_delta=738526, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.124:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29408442, succ_cluster_heartbeat_count=29408432, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774715049399002, v:0}, local_cluster_delta=95458402, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:20:10.507872] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.507888] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=14] handle task succ(id={id:1260}) [2024-08-16 10:20:10.507897] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.507905] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=7] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.507922] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.508143] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD86B2-0-0] [lt=35] [PALF STAT GROUP LOG INFO](palf_id=1261, self="192.168.0.128:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1770, avg_group_log_size=160) [2024-08-16 10:20:10.527233] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2780][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=47] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7079, mem_used=12478464) [2024-08-16 10:20:10.534648] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [2573][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=24] start do ls ha handler(ls_id_array_=[{id:1261}, {id:1260}, {id:1262}, {id:1079}]) [2024-08-16 10:20:10.552164] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:10.552273] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=25] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:10.552290] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:10.552297] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:10.552974] INFO [LIB] stat (utility.h:1197) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=22] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=710, this=0x7fa6bd55f3b0, extra_info="palf_id:1260") [2024-08-16 10:20:10.553007] INFO [LIB] stat (utility.h:1214) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=24] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=33, this=0x7fa7105fb3b0, extra_info="") [2024-08-16 10:20:10.553042] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=26] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=16, this=0x7fa6bd55f5e0, extra_info="palf_id:1260") [2024-08-16 10:20:10.553443] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F736F0-0-0] [lt=21] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=10, this=0x7fa6bd5556a0, extra_info="palf_id:1260") [2024-08-16 10:20:10.553465] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9F736F0-0-0] [lt=21] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=1257, this=0x7fa6bd5558d0, extra_info="palf_id:1260") [2024-08-16 10:20:10.553457] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:10.559126] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.559147] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:10.559158] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.559168] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.559182] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.560696] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD23849B7F-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6be52d470, extra_info="") [2024-08-16 10:20:10.588816] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 1.00, rt: 358.00) [2024-08-16 10:20:10.588847] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=26] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:10.588907] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=28] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.99, rt: 465.00) [2024-08-16 10:20:10.588924] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=14] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:6442450944, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:8}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=11717024, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-08-16 10:20:10.588954] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=23] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.588972] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=16] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.588980] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.588987] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.588994] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.588999] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.589003] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.589011] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.589027] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.589034] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.589039] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.589043] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.589049] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.589097] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=47] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.589104] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.589109] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [31397][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-08-16 10:20:10.591841] WDIAG [ARCHIVE] check_and_get_record_context_ (ob_ls_meta_recorder.cpp:206) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=9][errcode=-4018] get failed(ret=-4018, type={type:"schema_meta"}) [2024-08-16 10:20:10.594406] INFO [DATA_DICT] get_data_dict_meta_info_ (ob_data_dict_meta_info.cpp:482) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=23] get_data_dict_meta_info(tenant_id_=1002, base_scn={val:1720659999678271000, v:0}, record_count=119, valid_record_count=37) [2024-08-16 10:20:10.594494] WDIAG [ARCHIVE] get_ls_archive_progress (ob_archive_persist_mgr.cpp:168) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=22][errcode=-4023] archive progress not exist, need retry(ret=-4023, id={id:1}) [2024-08-16 10:20:10.594514] WDIAG [ARCHIVE] check_need_delay_ (ob_ls_meta_recorder.cpp:292) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=19][errcode=-4023] get ls arcchive progress failed(ret=-4023, id={id:1}) [2024-08-16 10:20:10.594522] INFO [ARCHIVE] handle (ob_ls_meta_recorder.cpp:190) [2567][T1002_ArcTimer][T1002][YB42C0A80080-00061CCC9241D0C5-0-0] [lt=7] check_need_delay_ return true, just wait(id={id:1}, task_type={type:"schema_meta"}, scn={val:1723770786620344000, v:0}) [2024-08-16 10:20:10.598714] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [31738][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=71] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:0, free_obj_hold_cnt:940, allocator used:3737440, allocator total:3859072}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:0, free_obj_hold_cnt:140, allocator used:9182740, allocator total:9182740}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():0, full allocator used:0, full allocator total:0}, tablets_mem=13041812, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:0, free_obj_hold_cnt:63, allocator used:124992, allocator total:130816}, pending_cnt=0, wait_gc_count=0, tablet count=0) [2024-08-16 10:20:10.599801] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=18] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:10.600085] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=27] Wash time detail, (compute_wash_size_time=117, refresh_score_time=67, wash_time=15) [2024-08-16 10:20:10.600673] INFO [CLOG] runTimerTask (ob_log_replay_service.cpp:159) [31746][T1_ReplayProces][T1][Y0-0000000000000000-0-0] [lt=20] dump tenant replay process(tenant_id=1, unreplayed_log_size(MB)=0, estimate_time(second)=0, replayed_log_size(MB)=0, last_replayed_log_size(MB)=0, round_cost_time(second)=10, pending_replay_log_size(MB)=0) [2024-08-16 10:20:10.602913] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF2B1-0-0] [lt=22] [PALF STAT COMMITTED LOG SIZE](palf_id=1079, self="192.168.0.128:2882", committed size=1492) [2024-08-16 10:20:10.602960] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF2B1-0-0] [lt=24] [PALF STAT LOG TASK TIME](palf_id=1079, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=13, avg_log_submit_to_flush_time=748, avg_log_submit_to_slide_time=100125) [2024-08-16 10:20:10.602979] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF2B1-0-0] [lt=17][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509958, log_task={header:{begin_lsn:{lsn:18544236542}, end_lsn:{lsn:18544236664}, log_id:53509958, min_scn:{val:1723774810393176000, v:0}, max_scn:{val:1723774810393176000, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544236420}, prev_proposal_id:7, committed_end_lsn:{lsn:18544236542}, data_checksum:1913811101, accum_checksum:122687568, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774810402718, freeze_ts:1723774810402719, submit_ts:1723774810402726, flushed_ts:1723774810403446, gen_to_freeze cost time:1, gen_to_submit cost time:8, submit_to_flush cost time:720}, fs_cb_begin_ts=1723774810602946, log_life_time=200228) [2024-08-16 10:20:10.603017] INFO [PALF] read (log_cache.cpp:100) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=14] [PALF STAT HOT CACHE HIT RATE](palf_id=1079, read_size=1492, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:10.603493] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=17] Cache wash clean map node details(ret=0, clean_node_count=89, clean_time=0, clean_start_pos=1572850, clean_num=62914) [2024-08-16 10:20:10.603750] INFO [LIB] stat (utility.h:1214) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=32, this=0x7fa7105fac70, extra_info="") [2024-08-16 10:20:10.607953] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.607973] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=18] handle task succ(id={id:1260}) [2024-08-16 10:20:10.607988] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=13] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.607998] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.608046] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=39] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.608103] INFO do_work (ob_rl_mgr.cpp:709) [31449][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=19] swc wakeup.(stat_period_=1000000, ready=false) [2024-08-16 10:20:10.618798] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [2556][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=25] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=154618822, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=60756, mem_used=137263104) [2024-08-16 10:20:10.645704] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=21] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=6) [2024-08-16 10:20:10.645845] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=23] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-08-16 10:20:10.645905] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=27] [tabletchange] task check ls(ls->get_ls_id()={id:1079}, tablet_persist_trigger=0) [2024-08-16 10:20:10.645933] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=24] [tabletchange] task check ls(ls->get_ls_id()={id:1260}, tablet_persist_trigger=0) [2024-08-16 10:20:10.645942] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=8] [tabletchange] task check ls(ls->get_ls_id()={id:1261}, tablet_persist_trigger=0) [2024-08-16 10:20:10.645947] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [tabletchange] task check ls(ls->get_ls_id()={id:1262}, tablet_persist_trigger=0) [2024-08-16 10:20:10.645951] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:228) [2561][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [tabletchange] succeed to gc_tablet(ret=0, ret="OB_SUCCESS", ls_cnt=4, times=11135) [2024-08-16 10:20:10.652246] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:10.652348] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:10.652364] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:10.652372] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:10.653539] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:10.659216] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.659245] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=26] handle task succ(id={id:1260}) [2024-08-16 10:20:10.659259] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.659284] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=22] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.659341] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=45] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.661427] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=581, this=0x7fa6be5373b0, extra_info="palf_id:1262") [2024-08-16 10:20:10.661511] INFO [LIB] stat (utility.h:1197) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=23] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=39, this=0x7fa6be5375e0, extra_info="palf_id:1262") [2024-08-16 10:20:10.701694] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [32165][T1_ComTimerTh][T1][Y0-0000000000000000-0-0] [lt=36] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-08-16 10:20:10.702557] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [31775][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=12] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-08-16 10:20:10.702590] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:101) [31775][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=24] [emptytablet] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=12639) [2024-08-16 10:20:10.708031] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.708053] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:10.708065] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.708115] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=48] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.708138] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.752326] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:10.752440] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=32] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=48, this=0x7fa6bd555470, extra_info="") [2024-08-16 10:20:10.752493] INFO [LIB] stat (utility.h:1197) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=34] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6bd542e50, extra_info="id:1260") [2024-08-16 10:20:10.752503] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:10.752518] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:10.752528] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:10.753528] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [2674][T1001_LogLoop][T1001][Y0-0000000000000000-0-0] [lt=17] LogLoopThread round_cost_time(us)(round_cost_time=3) [2024-08-16 10:20:10.753596] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80085-00061CCD236EE71F-0-0] [lt=26] [PALF STAT LOG TASK TIME](palf_id=1260, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=6, avg_log_gen_to_submit_time=70, avg_log_submit_to_flush_time=808, avg_log_submit_to_slide_time=1190) [2024-08-16 10:20:10.753613] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=28][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:10.759310] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.759341] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=28] handle task succ(id={id:1260}) [2024-08-16 10:20:10.759355] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.759366] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.759405] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=33] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.761806] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA744E2-0-0] [lt=30] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=10, this=0x7fa6be52d6a0, extra_info="palf_id:1262") [2024-08-16 10:20:10.761825] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA744E2-0-0] [lt=18] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=10014, this=0x7fa6be52d8d0, extra_info="palf_id:1262") [2024-08-16 10:20:10.803020] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF2BB-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fc6e9470, extra_info="") [2024-08-16 10:20:10.803112] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF2BB-0-0] [lt=42] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=12, this=0x7fa6fc6e96a0, extra_info="palf_id:1079") [2024-08-16 10:20:10.803126] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF2BB-0-0] [lt=13] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=91011, this=0x7fa6fc6e98d0, extra_info="palf_id:1079") [2024-08-16 10:20:10.803349] INFO [CLOG] push_log_replay_task (ob_replay_status.cpp:1140) [2586][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=18] submit pre barrier log success(log_type=17, ls_id={id:1079}, lsn={lsn:18544236964}, scn={val:1723774810655889001, v:0}, is_pre_barrier=true, is_post_barrier=false, log_size=24) [2024-08-16 10:20:10.803300] INFO [CLOG] do_replay_task_ (ob_log_replay_service.cpp:901) [2585][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=25] wait for barrier(ret=-4023, replay_status={ls_id_:{id:1079}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:7, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774810803104, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:11904, ret_code_:0}}, next_to_submit_lsn_:{lsn:18544236908}, next_to_submit_scn_:{val:1723774810655889001, v:0}, base_lsn_:{lsn:13018324992}, base_scn_:{val:1720509584063969004, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd005000, next_round_pread_size:3678208, curr_read_pos:56, curr_read_buf_start_pos:0, curr_read_buf_end_pos:112, log_storage_:{IteratorStorage:{start_lsn:{lsn:18544236908}, end_lsn:{lsn:18544237020}, read_buf:{buf_len_:2154496, buf_:0x7fa6cd005000}, block_size:67104768, log_storage_:0x7fa6fc6ef470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:56, prev_entry_scn:{val:1723774810655889001, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:24, scn_:{val:1723774810655889001, v:0}, data_checksum:2527417382, flag:1}}, init_mode_version:0, accumulate_checksum:2381958336, curr_entry_is_padding:0, padding_entry_size:56, padding_entry_scn:{val:1723774810655889001, v:0}}}}}, replay_task={ls_id_:{id:1079}, log_type:17, log_type:"RESERVED_SNAPSHOT", lsn_:{lsn:18544236964}, scn_:{val:1723774810655889001, v:0}, is_pre_barrier_:true, is_post_barrier_:false, read_log_size_:24, replay_hint_:1723774810712429, first_handle_ts_:1723774810803296, replay_cost_:-1, retry_cost_:-1, read_log_buf_:0x7fa5a4190e60, decompression_buf_:null, has_decompressed_:false, decompressed_log_size_:0, base_header_len_:16}) [2024-08-16 10:20:10.803514] INFO [STORAGE] inner_update_reserved_snapshot (ob_ls_reserved_snapshot_mgr.cpp:288) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=20] success to update reserved snapshot(ret=0, ls_id={id:1079}, min_reserved_snapshot_=1723773000638406000) [2024-08-16 10:20:10.803540] INFO [CLOG] do_replay_task_ (ob_log_replay_service.cpp:919) [2591][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=23] pre barrier log replay succ(replay_task={ls_id_:{id:1079}, log_type:17, log_type:"RESERVED_SNAPSHOT", lsn_:{lsn:18544236964}, scn_:{val:1723774810655889001, v:0}, is_pre_barrier_:true, is_post_barrier_:false, read_log_size_:24, replay_hint_:1723774810712429, first_handle_ts_:1723774810803494, replay_cost_:43, retry_cost_:-1, read_log_buf_:0x7fa5a4190e70, decompression_buf_:null, has_decompressed_:false, decompressed_log_size_:0, base_header_len_:16}, replay_status={ls_id_:{id:1079}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774810803104, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:11904, ret_code_:0}}, next_to_submit_lsn_:{lsn:18544237020}, next_to_submit_scn_:{val:1723774810655889002, v:0}, base_lsn_:{lsn:13018324992}, base_scn_:{val:1720509584063969004, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd005000, next_round_pread_size:3678208, curr_read_pos:112, curr_read_buf_start_pos:0, curr_read_buf_end_pos:112, log_storage_:{IteratorStorage:{start_lsn:{lsn:18544236908}, end_lsn:{lsn:18544237020}, read_buf:{buf_len_:2154496, buf_:0x7fa6cd005000}, block_size:67104768, log_storage_:0x7fa6fc6ef470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774810655889001, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:24, scn_:{val:1723774810655889001, v:0}, data_checksum:2527417382, flag:1}}, init_mode_version:0, accumulate_checksum:2381958336, curr_entry_is_padding:0, padding_entry_size:56, padding_entry_scn:{val:1723774810655889001, v:0}}}}}, replay_queue_idx=13, ret=0) [2024-08-16 10:20:10.804001] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=36] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=805847040, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:10.804286] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=35] Wash time detail, (compute_wash_size_time=126, refresh_score_time=87, wash_time=16) [2024-08-16 10:20:10.807689] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=16] Cache wash clean map node details(ret=0, clean_node_count=91, clean_time=0, clean_start_pos=1635764, clean_num=62914) [2024-08-16 10:20:10.808102] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=17] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.808123] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=19] handle task succ(id={id:1260}) [2024-08-16 10:20:10.808133] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.808143] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.808156] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=5] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.808204] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=70] get_max_applied_scn(scn={val:1723774810432890002, v:0}, this={ls_id_:{id:1260}, role_:1, proposal_id_:29, palf_committed_end_lsn_:{lsn:8828157143}, last_check_scn_:{val:1723774810432890003, v:0}, max_applied_cb_scn_:{val:1723774810432890002, v:0}}) [2024-08-16 10:20:10.808236] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=26] get_min_unreplayed_log_info(lsn={lsn:8469424174}, scn={val:1723617929041268003, v:0}, this={ls_id_:{id:1260}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723617929662670, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:2710542, ret_code_:0}}, next_to_submit_lsn_:{lsn:8469424174}, next_to_submit_scn_:{val:1723617929041268003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516550107534003, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69fc05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8469424052}, end_lsn:{lsn:8469424174}, read_buf:{buf_len_:3682304, buf_:0x7fa69fc05000}, block_size:67104768, log_storage_:0x7fa6bd55b470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723617929041268002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723617929041268002, v:0}, data_checksum:3690972251, flag:0}}, init_mode_version:0, accumulate_checksum:2828504552, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723617929041268002, v:0}}}}}) [2024-08-16 10:20:10.808322] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=75] get_max_applied_scn(scn={val:1723617908966009002, v:0}, this={ls_id_:{id:1261}, role_:2, proposal_id_:4, palf_committed_end_lsn_:{lsn:32276646160}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1723617908966009002, v:0}}) [2024-08-16 10:20:10.808385] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=15] get_min_unreplayed_log_info(lsn={lsn:32601576512}, scn={val:1723774810411382003, v:0}, this={ls_id_:{id:1261}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774810709258, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:344950, ret_code_:0}}, next_to_submit_lsn_:{lsn:32601576512}, next_to_submit_scn_:{val:1723774810411382003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516552080113001, v:0}, iterator_:{iterator_impl:{buf_:0x7fa6cd805000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:32601576390}, end_lsn:{lsn:32601576512}, read_buf:{buf_len_:3682304, buf_:0x7fa6cd805000}, block_size:67104768, log_storage_:0x7fa6fe299470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774810411382002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774810411382002, v:0}, data_checksum:1657047228, flag:1}}, init_mode_version:0, accumulate_checksum:3366193511, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774810411382002, v:0}}}}}) [2024-08-16 10:20:10.808437] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=40] get_max_applied_scn(scn={val:0, v:0}, this={ls_id_:{id:1262}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:0, v:0}}) [2024-08-16 10:20:10.808478] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [2557][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=17] get_min_unreplayed_log_info(lsn={lsn:8394862234}, scn={val:1723774810269649005, v:0}, this={ls_id_:{id:1262}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1723774810761795, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10679383, ret_code_:0}}, next_to_submit_lsn_:{lsn:8394862234}, next_to_submit_scn_:{val:1723774810269649005, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1720516551920929002, v:0}, iterator_:{iterator_impl:{buf_:0x7fa69ec05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:8394862112}, end_lsn:{lsn:8394862234}, read_buf:{buf_len_:3682304, buf_:0x7fa69ec05000}, block_size:67104768, log_storage_:0x7fa6be533470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1723774810269649004, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1723774810269649004, v:0}, data_checksum:2892311396, flag:0}}, init_mode_version:0, accumulate_checksum:77180370, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1723774810269649004, v:0}}}}}) [2024-08-16 10:20:10.845818] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=36] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=5) [2024-08-16 10:20:10.852397] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:10.852542] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=28] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:10.852559] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:10.852566] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:10.853326] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [2427][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=19] [PALF STAT COMMITTED LOG SIZE](palf_id=1260, self="192.168.0.128:2882", committed size=1220) [2024-08-16 10:20:10.853694] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=13][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:10.859396] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.859428] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=30] handle task succ(id={id:1260}) [2024-08-16 10:20:10.859439] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.859450] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.859462] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.860425] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=23] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1001, server_version={version:{val:1723774805968500000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0}, server_version_delta=4891921, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="192.168.0.124:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=29431429, succ_cluster_heartbeat_count=29431417, cluster_heartbeat_interval=50000, local_cluster_version={val:1723774715706314000, v:0}, local_cluster_delta=95154107, force_self_check=false, weak_read_refresh_interval=100000) [2024-08-16 10:20:10.903180] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF2BF-0-0] [lt=6][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509962, log_task={header:{begin_lsn:{lsn:18544237020}, end_lsn:{lsn:18544237142}, log_id:53509962, min_scn:{val:1723774810655889002, v:0}, max_scn:{val:1723774810655889002, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544236908}, prev_proposal_id:7, committed_end_lsn:{lsn:18544237020}, data_checksum:1913811101, accum_checksum:1759850925, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774810803010, freeze_ts:1723774810803010, submit_ts:1723774810803066, flushed_ts:1723774810803755, gen_to_freeze cost time:0, gen_to_submit cost time:56, submit_to_flush cost time:689}, fs_cb_begin_ts=1723774810903152, log_life_time=100142) [2024-08-16 10:20:10.903891] INFO [LIB] stat (utility.h:1197) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=29] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=676, this=0x7fa6fc6f33b0, extra_info="palf_id:1079") [2024-08-16 10:20:10.903914] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2429][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=21] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1079, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1604) [2024-08-16 10:20:10.908181] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.908204] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=21] handle task succ(id={id:1260}) [2024-08-16 10:20:10.908216] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.908240] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=23] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.908257] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.935026] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [31538][TsMgr][T0][Y0-0000000000000000-0-0] [lt=16] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-08-16 10:20:10.948529] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=8] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1002) [2024-08-16 10:20:10.948561] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=24] start lib cache evict(tenant_id=1002, mem_hold=279150592, mem_limit=322122545, cache_obj_num=644, cache_node_num=575) [2024-08-16 10:20:10.948582] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=17] end lib cache evict(tenant_id=1002, cache_evict_num=0, mem_hold=279150592, mem_limit=322122545, cache_obj_num=644, cache_node_num=575) [2024-08-16 10:20:10.948593] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=10] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:10.951548] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [31787][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=18] tx gc loop thread is running(MTL_ID()=1) [2024-08-16 10:20:10.952478] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:10.952600] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=27] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:10.952617] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:10.952624] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:10.951989] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=21] Dumping All Cache Objs(alloc_obj_list.count()=644, alloc_obj_list=[{obj_id:197058, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179184}, {obj_id:246944, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:166013}, {obj_id:246946, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175596}, {obj_id:199478, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205804}, {obj_id:249042, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:250289, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174834}, {obj_id:250293, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187797}, {obj_id:250294, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158389}, {obj_id:252064, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179845}, {obj_id:252089, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252117, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188403}, {obj_id:252210, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252318, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191338}, {obj_id:252323, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182962}, {obj_id:252333, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252334, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252336, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252340, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252341, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:252345, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221400}, {obj_id:252346, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284685}, {obj_id:252349, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181199}, {obj_id:252359, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:356776}, {obj_id:252372, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180356}, {obj_id:252377, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252386, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252404, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:382127}, {obj_id:252405, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252406, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:252409, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230382}, {obj_id:252417, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277038}, {obj_id:252425, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158366}, {obj_id:252428, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252440, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213028}, {obj_id:252445, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178214}, {obj_id:252447, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:252458, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:941831}, {obj_id:252459, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252464, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157801}, {obj_id:252475, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176302}, {obj_id:252485, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181511}, {obj_id:252488, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203010}, {obj_id:252489, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197827}, {obj_id:252495, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181747}, {obj_id:252502, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:705639}, {obj_id:252503, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:355214}, {obj_id:252511, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294158}, {obj_id:252513, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217004}, {obj_id:252540, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256131}, {obj_id:252541, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294112}, {obj_id:252554, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219188}, {obj_id:252569, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203562}, {obj_id:252573, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174535}, {obj_id:252586, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179957}, {obj_id:252587, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:254547}, {obj_id:252603, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294123}, {obj_id:252660, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196529}, {obj_id:252664, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294182}, {obj_id:252670, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292094}, {obj_id:252671, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294048}, {obj_id:252673, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211150}, {obj_id:252699, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202759}, {obj_id:252702, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252720, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157744}, {obj_id:252721, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:273791}, {obj_id:252730, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189375}, {obj_id:252749, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:399910}, {obj_id:252750, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:314035}, {obj_id:252764, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299083}, {obj_id:252765, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179021}, {obj_id:252769, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:355879}, {obj_id:252771, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188178}, {obj_id:252786, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237378}, {obj_id:252841, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177408}, {obj_id:252849, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252858, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180522}, {obj_id:252859, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180179}, {obj_id:252862, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6708112}, {obj_id:252864, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252865, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232017}, {obj_id:252868, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232266}, {obj_id:252869, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220840}, {obj_id:252939, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252955, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175994}, {obj_id:252962, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252964, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:349147}, {obj_id:252975, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252980, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252994, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252995, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:252998, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253001, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253005, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253006, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253007, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253008, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294264}, {obj_id:253009, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292151}, {obj_id:253015, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298926}, {obj_id:253022, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301178}, {obj_id:253025, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298998}, {obj_id:253035, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298892}, {obj_id:253039, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:351399}, {obj_id:253040, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292087}, {obj_id:253047, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298992}, {obj_id:253050, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:288707}, {obj_id:253053, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204401}, {obj_id:253059, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208911}, {obj_id:253060, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211382}, {obj_id:253061, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200421}, {obj_id:253065, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223579}, {obj_id:253071, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290322}, {obj_id:253072, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250919}, {obj_id:253087, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253090, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:323458}, {obj_id:253091, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272373}, {obj_id:253094, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200393}, {obj_id:253095, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187302}, {obj_id:253096, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206572}, {obj_id:253097, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190787}, {obj_id:253098, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177031}, {obj_id:253102, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272582}, {obj_id:253107, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272526}, {obj_id:253113, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213908}, {obj_id:253114, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201513}, {obj_id:253115, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185332}, {obj_id:253122, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240210}, {obj_id:253124, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245608}, {obj_id:253127, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:372694}, {obj_id:253137, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1957477}, {obj_id:253169, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:270222}, {obj_id:253170, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:272447}, {obj_id:253174, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:795070}, {obj_id:253234, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:283358}, {obj_id:253282, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253312, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286729}, {obj_id:253341, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253444, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157703}, {obj_id:253450, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186097}, {obj_id:253451, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:287939}, {obj_id:253452, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258575}, {obj_id:253454, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277108}, {obj_id:253457, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218998}, {obj_id:253458, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:285701}, {obj_id:253459, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253468, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181399}, {obj_id:253471, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:874259}, {obj_id:253473, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232858}, {obj_id:253476, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253477, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253478, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207070}, {obj_id:253480, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250296}, {obj_id:253485, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253486, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201045}, {obj_id:253487, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201415}, {obj_id:253488, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202980}, {obj_id:253489, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225769}, {obj_id:253490, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201728}, {obj_id:253491, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197763}, {obj_id:253497, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253500, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253506, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:11947420}, {obj_id:253507, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228298}, {obj_id:253508, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176153}, {obj_id:253509, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223501}, {obj_id:253510, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176542}, {obj_id:253511, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:253512, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218568}, {obj_id:253513, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177328}, {obj_id:253514, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177907}, {obj_id:253515, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184844}, {obj_id:253516, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253517, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177168}, {obj_id:253518, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:253519, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253520, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:253521, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:253522, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:253523, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253524, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253525, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253526, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253527, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:162334}, {obj_id:253529, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253530, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181547}, {obj_id:253531, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181340}, {obj_id:253532, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192983}, {obj_id:253533, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253534, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186898}, {obj_id:253535, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180995}, {obj_id:253536, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253537, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253538, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:253539, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253540, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223911}, {obj_id:253541, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253542, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221984}, {obj_id:253543, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186900}, {obj_id:253544, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213278}, {obj_id:253545, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6705734}, {obj_id:253546, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6739530}, {obj_id:253547, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202652}, {obj_id:253548, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253549, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:236673}, {obj_id:253550, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253551, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253553, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253554, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:367141}, {obj_id:253555, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189381}, {obj_id:253556, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253557, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253558, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205876}, {obj_id:253559, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253560, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189377}, {obj_id:253561, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253562, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237070}, {obj_id:253564, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205271}, {obj_id:253565, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238959}, {obj_id:253566, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292100}, {obj_id:253567, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:287824}, {obj_id:253568, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:6741367}, {obj_id:253570, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187799}, {obj_id:253571, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253572, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175560}, {obj_id:253573, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175558}, {obj_id:253574, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:160948}, {obj_id:253576, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253578, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253579, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253580, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235030}, {obj_id:253581, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234962}, {obj_id:253582, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200857}, {obj_id:253583, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253584, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175142}, {obj_id:253585, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174672}, {obj_id:253588, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292043}, {obj_id:253589, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253590, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253591, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253594, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292025}, {obj_id:253595, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253596, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253601, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:383372}, {obj_id:253605, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292063}, {obj_id:253608, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290453}, {obj_id:253610, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253611, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:369179}, {obj_id:253614, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282531}, {obj_id:253615, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292282}, {obj_id:253616, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253617, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197701}, {obj_id:253618, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:476477}, {obj_id:253621, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297717}, {obj_id:253628, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:244896}, {obj_id:253630, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203464}, {obj_id:253631, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196891}, {obj_id:253632, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:253633, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186088}, {obj_id:253634, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181148}, {obj_id:253635, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253638, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:268654}, {obj_id:253640, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201594}, {obj_id:253641, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:265278}, {obj_id:253642, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290670}, {obj_id:253645, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253649, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292134}, {obj_id:253650, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206516}, {obj_id:253653, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290821}, {obj_id:253655, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238325}, {obj_id:253658, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198080}, {obj_id:253659, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216604}, {obj_id:253660, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178528}, {obj_id:253661, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:243178}, {obj_id:253664, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291977}, {obj_id:253665, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216113}, {obj_id:253666, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242569}, {obj_id:253667, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253668, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207760}, {obj_id:253669, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253672, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292096}, {obj_id:253677, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178509}, {obj_id:253681, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292081}, {obj_id:253682, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182898}, {obj_id:253683, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225569}, {obj_id:253691, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208341}, {obj_id:253692, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211962}, {obj_id:253694, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:379579}, {obj_id:253695, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185463}, {obj_id:253696, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253697, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253698, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194454}, {obj_id:253699, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185281}, {obj_id:253700, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277153}, {obj_id:253701, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196647}, {obj_id:253702, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220331}, {obj_id:253703, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253704, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228523}, {obj_id:253705, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184067}, {obj_id:253706, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175893}, {obj_id:253707, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189895}, {obj_id:253708, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251032}, {obj_id:253709, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189423}, {obj_id:253710, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:334817}, {obj_id:253711, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189681}, {obj_id:253712, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209440}, {obj_id:253713, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188174}, {obj_id:253714, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253715, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253716, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174832}, {obj_id:253723, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:932120}, {obj_id:253724, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:262469}, {obj_id:253725, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:332574}, {obj_id:253726, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240754}, {obj_id:253727, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210719}, {obj_id:253728, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253729, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:253730, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185730}, {obj_id:253731, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250192}, {obj_id:253732, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177398}, {obj_id:253733, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191786}, {obj_id:253734, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177488}, {obj_id:253735, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276369}, {obj_id:253736, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219016}, {obj_id:253737, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:510255}, {obj_id:253740, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197323}, {obj_id:253743, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234930}, {obj_id:253747, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210144}, {obj_id:253749, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253750, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253751, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227821}, {obj_id:253753, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:302184}, {obj_id:253754, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253755, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183280}, {obj_id:253756, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253757, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282524}, {obj_id:253758, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200791}, {obj_id:253759, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:384669}, {obj_id:253760, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:382379}, {obj_id:253761, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:873919}, {obj_id:253762, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:870651}, {obj_id:253763, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:516499}, {obj_id:253764, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253765, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253766, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204396}, {obj_id:253767, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222427}, {obj_id:253768, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253769, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221512}, {obj_id:253770, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:554188}, {obj_id:253771, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247245}, {obj_id:253772, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253773, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196589}, {obj_id:253774, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187792}, {obj_id:253775, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198057}, {obj_id:253776, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279996}, {obj_id:253777, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221914}, {obj_id:253779, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180817}, {obj_id:253780, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179884}, {obj_id:253782, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225027}, {obj_id:253784, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250315}, {obj_id:253791, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292016}, {obj_id:253792, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291532}, {obj_id:253793, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180590}, {obj_id:253796, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250098}, {obj_id:253797, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180228}, {obj_id:253798, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176856}, {obj_id:253799, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195067}, {obj_id:253800, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253801, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177751}, {obj_id:253802, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208961}, {obj_id:253805, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240180}, {obj_id:253808, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256449}, {obj_id:253809, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:449159}, {obj_id:253810, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188388}, {obj_id:253811, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199543}, {obj_id:253812, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195231}, {obj_id:253813, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253814, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253815, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214158}, {obj_id:253816, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253817, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:288727}, {obj_id:253820, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:291994}, {obj_id:253821, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235062}, {obj_id:253827, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175823}, {obj_id:253828, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174483}, {obj_id:253829, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217784}, {obj_id:253830, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177814}, {obj_id:253831, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:444256}, {obj_id:253832, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195622}, {obj_id:253833, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253834, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:497014}, {obj_id:253835, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253836, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:261672}, {obj_id:253837, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:435156}, {obj_id:253838, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:516719}, {obj_id:253839, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200167}, {obj_id:253840, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271961}, {obj_id:253841, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194596}, {obj_id:253842, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205257}, {obj_id:253843, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174397}, {obj_id:253845, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253846, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253847, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253848, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224798}, {obj_id:253849, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223872}, {obj_id:253850, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224553}, {obj_id:253851, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221103}, {obj_id:253852, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219500}, {obj_id:253853, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219425}, {obj_id:253854, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253855, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240470}, {obj_id:253856, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253857, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253858, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220267}, {obj_id:253859, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220212}, {obj_id:253861, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:290476}, {obj_id:253864, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:547504}, {obj_id:253865, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233724}, {obj_id:253866, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218340}, {obj_id:253867, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197559}, {obj_id:253868, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253869, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217547}, {obj_id:253870, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235957}, {obj_id:253871, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237233}, {obj_id:253872, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:280008}, {obj_id:253873, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208674}, {obj_id:253874, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190144}, {obj_id:253875, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240637}, {obj_id:253876, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187604}, {obj_id:253878, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241654}, {obj_id:253879, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253880, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253881, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176921}, {obj_id:253882, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188072}, {obj_id:253883, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:246455}, {obj_id:253884, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176040}, {obj_id:253886, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253887, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:253890, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, add [2024-08-16 10:20:10.952861] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [2455][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=868] schedule next cache evict task(evict_interval=5000000) [2024-08-16 10:20:10.953768] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=29][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:10.959479] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:10.959501] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:10.959511] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:10.959522] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=10] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:10.959550] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=14] try consume task status succ(id={id:1260}) [2024-08-16 10:20:10.971801] INFO [CLOG] is_in_sync (ob_log_handler.cpp:546) [31817][T1_L5_G0][T1002][YB42C0A8007C-000616D3CE4B970D-0-0] [lt=24] is_in_sync(ret=0, id=1079, is_log_sync=true, is_need_rebuild=false) [2024-08-16 10:20:10.971879] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A8007C-000616D3CE4B970D-0-0] [lt=23][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:10.998794] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=25] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:10.998824] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [31388][ServerGTimer][T1002][Y0-0000000000000000-0-0] [lt=24] get_disk_usage(ret=0, capacity(MB):=19353, used(MB):=15477) [2024-08-16 10:20:10.998963] WDIAG load_file_to_string (utility.h:659) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] read /sys/class/net/eth0/speed failed, errno 22 [2024-08-16 10:20:10.998986] WDIAG get_ethernet_speed (utility.cpp:580) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=16][errcode=-4000] load file /sys/class/net/eth0/speed failed, ret -4000 [2024-08-16 10:20:10.999007] WDIAG [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2717) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=18][errcode=-4000] cannot get Ethernet speed, use default(tmp_ret=0, devname="eth0") [2024-08-16 10:20:10.999020] WDIAG [SERVER] runTimerTask (ob_server.cpp:3239) [31388][ServerGTimer][T0][Y0-0000000000000000-0-0] [lt=11][errcode=-4000] ObRefreshNetworkSpeedTask reload bandwidth throttle limit failed(ret=-4000, ret="OB_ERROR") [2024-08-16 10:20:11.008276] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:11.008318] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=40] handle task succ(id={id:1260}) [2024-08-16 10:20:11.008330] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:11.008312] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=20] Wash compute wash size(is_wash_valid=true, sys_total_wash_size=803749888, global_cache_size=95715328, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1, 1001, 1002]) [2024-08-16 10:20:11.008339] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:11.008354] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=6] try consume task status succ(id={id:1260}) [2024-08-16 10:20:11.008610] INFO [COMMON] wash (ob_kvcache_store.cpp:342) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=29] Wash time detail, (compute_wash_size_time=169, refresh_score_time=74, wash_time=16) [2024-08-16 10:20:11.011874] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [31419][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=22] Cache wash clean map node details(ret=0, clean_node_count=81, clean_time=0, clean_start_pos=1698678, clean_num=62914) [2024-08-16 10:20:11.013100] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [31776][T1_LSArchiveMgr][T1][YB42C0A80080-00061CCC8D11D0C5-0-0] [lt=28] gc stale ls task succ [2024-08-16 10:20:11.013212] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=5] gc stale ls task succ [2024-08-16 10:20:11.013240] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=19] ls not need do archive on this server(ls_id={id:1079}) [2024-08-16 10:20:11.013252] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=7] ls not need do archive on this server(ls_id={id:1261}) [2024-08-16 10:20:11.013265] INFO [ARCHIVE] add_ls_task_ (ob_ls_mgr.cpp:409) [2563][T1002_LSArchive][T1002][YB42C0A80080-00061CCC9201D0C5-0-0] [lt=11] ls not need do archive on this server(ls_id={id:1262}) [2024-08-16 10:20:11.016886] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31540][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:11.016973] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [31539][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-08-16 10:20:11.031490] WDIAG [SQL.ENG] inner_get_next_row (ob_px_dist_transmit_op.cpp:73) [31817][T1_L5_G0][T1][YB42C0A8007E-00061EFF89AB7947-0-0] [lt=15][errcode=-4008] fail to exec ObPxTransmitOp::inner_get_next_row()(ret=-4008) [2024-08-16 10:20:11.033700] INFO [PALF] handle_learner_keepalive_req (log_config_mgr.cpp:2720) [13655][T1002_L0_G1][T1002][YB42C0A80083-000616E4699E2C38-0-0] [lt=75] handle_learner_keepalive_req success(ret=0, ret="OB_SUCCESS", palf_id=1079, self="192.168.0.128:2882", parent={server:"192.168.0.131:2882", region:"sys_region", register_time_us:1720516633177815, keepalive_ts:-1}, parent_keepalive_time_us=1723774811033658) [2024-08-16 10:20:11.036918] WDIAG [SQL.ENG] inner_get_next_row (ob_px_dist_transmit_op.cpp:73) [31817][T1_L5_G0][T1][YB42C0A8007E-00061EFF89AB7947-0-0] [lt=30][errcode=-4008] fail to exec ObPxTransmitOp::inner_get_next_row()(ret=-4008) [2024-08-16 10:20:11.045937] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [31420][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=32] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=4) [2024-08-16 10:20:11.046116] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31842][BlackListServic][T1][YB42C0A80080-00061CCCC6B436C4-0-0] [lt=39] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12239, add_timestamp:1723774811046100}) [2024-08-16 10:20:11.046167] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [31842][BlackListServic][T1][YB42C0A80080-00061CCCC6B436C4-0-0] [lt=22] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12287, add_timestamp:1723774811046165}) [2024-08-16 10:20:11.046161] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670D-0-0] [lt=38] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12239, locations=[{table_id:12239, partition_id:12239, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046141, sql_renew_time:1723774811046141, is_mark_fail:false}]) [2024-08-16 10:20:11.046190] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670D-0-0] [lt=27] success to process renew task(task={tenant_id:1, table_id:12239, add_timestamp:1723774811046100}, locations=[{table_id:12239, partition_id:12239, partition_cnt:0, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046141, sql_renew_time:1723774811046141, is_mark_fail:false}]) [2024-08-16 10:20:11.046213] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670E-0-0] [lt=13] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=12287, vtable_type={tenant_id:1, type:4}) [2024-08-16 10:20:11.046693] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670E-0-0] [lt=17] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12287, locations=[{table_id:12287, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:11.046746] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [31510][VTblLocAsyncUp0][T0][YB42C0A80080-00061CCCC967670E-0-0] [lt=52] success to process renew task(task={tenant_id:1, table_id:12287, add_timestamp:1723774811046165}, locations=[{table_id:12287, partition_id:0, partition_cnt:9, replica_locations:[{server:"192.168.0.124:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:1, partition_cnt:9, replica_locations:[{server:"192.168.0.125:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:2, partition_cnt:9, replica_locations:[{server:"192.168.0.126:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:3, partition_cnt:9, replica_locations:[{server:"192.168.0.127:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:4, partition_cnt:9, replica_locations:[{server:"192.168.0.128:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:5, partition_cnt:9, replica_locations:[{server:"192.168.0.129:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:6, partition_cnt:9, replica_locations:[{server:"192.168.0.131:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:7, partition_cnt:9, replica_locations:[{server:"192.168.0.133:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:8, partition_cnt:9, replica_locations:[{server:"192.168.0.134:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1723774811046679, sql_renew_time:0, is_mark_fail:false}]) [2024-08-16 10:20:11.052546] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:11.052671] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=21] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:11.052690] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:11.052698] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:11.053857] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-08-16 10:20:11.054186] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [31817][T1_L5_G0][T1][YB42C0A80080-00061CCCC6B436C4-0-0] [lt=29][errcode=-4008] execute fail(ret=-4008) [2024-08-16 10:20:11.057474] INFO [STORAGE.TRANS] run1 (ob_black_list.cpp:194) [31842][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=15] ls blacklist refresh finish(cost_time=12016) [2024-08-16 10:20:11.059562] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=16] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:11.059584] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=20] handle task succ(id={id:1260}) [2024-08-16 10:20:11.059592] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=6] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:11.059601] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=8] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:11.059617] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2650][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC92D1D0C5-0-0] [lt=7] try consume task status succ(id={id:1260}) [2024-08-16 10:20:11.060623] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [2781][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=57] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-08-16 10:20:11.065716] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-08-16 10:20:11.065737] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [2440][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=19][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-08-16 10:20:11.103432] INFO eloop_run (eloop.c:143) [31439][pnio1][T0][Y0-0000000000000000-0-0] [lt=29] PNIO [ratelimit] time: 1723774811103429, bytes: 1496438233087, bw: 0.167038 MB/s, add_ts: 1000930, add_bytes: 175315 [2024-08-16 10:20:11.103520] WDIAG [PALF] sliding_cb (log_sliding_window.cpp:2272) [13655][T1002_L0_G1][T1002][YB42C0A8007C-000616D3F7FCF2CB-0-0] [lt=38][errcode=-4389] log_task life cost too much time(palf_id=1079, self="192.168.0.128:2882", log_id=53509964, log_task={header:{begin_lsn:{lsn:18544237264}, end_lsn:{lsn:18544237386}, log_id:53509964, min_scn:{val:1723774810894065001, v:0}, max_scn:{val:1723774810894065001, v:0}, data_len:66, proposal_id:7, prev_lsn:{lsn:18544237142}, prev_proposal_id:7, committed_end_lsn:{lsn:18544237264}, data_checksum:1913811101, accum_checksum:707738148, is_padding_log:false, is_raw_write:false}, state_map:{val:58}, ref_cnt:0, gen_ts:1723774811003260, freeze_ts:1723774811003260, submit_ts:1723774811003267, flushed_ts:1723774811004089, gen_to_freeze cost time:0, gen_to_submit cost time:7, submit_to_flush cost time:822}, fs_cb_begin_ts=1723774811103500, log_life_time=100240) [2024-08-16 10:20:11.103611] INFO [LIB] stat (utility.h:1197) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=12] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=61, stat_interval=1000000, avg (count/cost)=103, this=0x7fa7136688f8, extra_info="") [2024-08-16 10:20:11.103643] INFO [LIB] stat (utility.h:1197) [2590][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=30] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=61, stat_interval=1000000, avg (count/cost)=8, this=0x7fa713668b28, extra_info="") [2024-08-16 10:20:11.108372] INFO [ARCHIVE] submit_residual_log_fetch_task_ (ob_archive_fetcher.cpp:988) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=11] submit residual log fetch task succ(&task=0x7fa6a56f75b0) [2024-08-16 10:20:11.108399] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:341) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=25] handle task succ(id={id:1260}) [2024-08-16 10:20:11.108413] INFO [ARCHIVE] get_sorted_fetch_log (ob_ls_task.cpp:187) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=12] no task exist, just skip(tenant_id_=1002, id_={id:1260}, tmp_task=NULL) [2024-08-16 10:20:11.108424] INFO [ARCHIVE] try_consume_fetch_log_ (ob_archive_fetcher.cpp:896) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] no log fetch task exist, just skip(ret=0, id={id:1260}, task_exist=false) [2024-08-16 10:20:11.108446] INFO [ARCHIVE] handle_single_task_ (ob_archive_fetcher.cpp:349) [2565][T1002_ArcFetche][T1002][YB42C0A80080-00061CCC9221D0C5-0-0] [lt=9] try consume task status succ(id={id:1260}) [2024-08-16 10:20:11.108481] INFO [LIB] stat (utility.h:1197) [13655][T1002_L0_G1][T1002][YB42C0A8007D-00061CCCA9BD86BE-0-0] [lt=61] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x7fa6fe293470, extra_info="") [2024-08-16 10:20:11.109614] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [13655][T1002_L0_G1][T1002][YB42C0A80080-00061CCCCCA744ED-0-0] [lt=13] [PALF STAT LOG TASK TIME](palf_id=1261, self="192.168.0.128:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=7, avg_log_submit_to_flush_time=660, avg_log_submit_to_slide_time=1089) [2024-08-16 10:20:11.109633] INFO [PALF] read (log_cache.cpp:100) [2588][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT HOT CACHE HIT RATE](palf_id=1261, read_size=1667, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-08-16 10:20:11.140680] INFO eloop_run (eloop.c:143) [31440][pnio2][T0][Y0-0000000000000000-0-0] [lt=35] PNIO [ratelimit] time: 1723774811140677, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007584, add_bytes: 0 [2024-08-16 10:20:11.150194] INFO [PALF] runTimerTask (block_gc_timer_task.cpp:101) [2672][T1001_PalfGC][T1001][Y0-0000000000000000-0-0] [lt=26] BlockGCTimerTask success(ret=0, cost_time_us=10, palf_env_impl_={IPalfEnvImpl:{IPalfEnvImpl:"Dummy"}, self:"192.168.0.128:2882", log_dir:"/home/admin/oceanbase/store/ep_test_cluster/clog/tenant_1001", disk_options_wrapper:{disk_opts_for_stopping_writing:{log_disk_size(MB):2150, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):2150, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:4}, log_alloc_mgr_:{flying_log_task:0, flying_meta_task:0}}) [2024-08-16 10:20:11.152639] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1079}) [2024-08-16 10:20:11.152754] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] [PALF STAT GROUP LOG INFO](palf_id=1260, self="192.168.0.128:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-08-16 10:20:11.152793] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=33] [PALF STAT APPEND DATA SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, append size=1220) [2024-08-16 10:20:11.152806] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1260}) [2024-08-16 10:20:11.152825] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1261}) [2024-08-16 10:20:11.152833] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [2574][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1262}) [2024-08-16 10:20:11.153654] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [2430][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1260, self:"192.168.0.128:2882", has_set_deleted:false}, accum_size=1220) [2024-08-16 10:20:11.153935] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [2555][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=27][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT")