[2024-08-09 16:17:30.462494] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:635) [32458][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=53] Cache wash clean map node details(ret=0, clean_node_count=0, clean_time=1933, clean_start_pos=251656, clean_num=62914) [2024-08-09 16:17:30.468514] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C686AD22-0-0] [lt=29][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.468970] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C686AD22-0-0] [lt=10][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.469827] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C686AD22-0-0] [lt=7][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.470240] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C686AD22-0-0] [lt=43][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.470506] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C686AD22-0-0] [lt=8][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.479538] WDIAG [STORAGE.TRANS] get_cluster_service_master_ (ob_tenant_weak_read_service.cpp:291) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=5][errcode=-4076] tenant schema is not ready, need wait(ret=-4076, ret="OB_NEED_WAIT", superior_tenant_id=1) [2024-08-09 16:17:30.479554] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=15][errcode=-4076] tenant weak read service cluster heartbeat RPC fail(ret=-4076, rcode={code:-4076, msg:"post cluster heartbeat rpc failed, tenant_id=1", warnings:[]}, tenant_id_=1, dst="ip2:2882", cluster_service_tablet_id={id:226}) [2024-08-09 16:17:30.479571] WDIAG [STORAGE.TRANS] do_cluster_heartbeat_ (ob_tenant_weak_read_service.cpp:869) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] post cluster heartbeat rpc fail(ret=-4076, ret="OB_NEED_WAIT", tenant_id_=1, local_server_version={val:18446744073709551615, v:3}, valid_part_count=0, total_part_count=0, generate_timestamp=1723191450479535) [2024-08-09 16:17:30.479580] WDIAG [STORAGE.TRANS] do_cluster_heartbeat_ (ob_tenant_weak_read_service.cpp:879) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] tenant weak read service do cluster heartbeat fail(ret=-4076, ret="OB_NEED_WAIT", tenant_id_=1, last_post_cluster_heartbeat_tstamp_=1723191450379436, cluster_heartbeat_interval_=1000000, cluster_service_tablet_id={id:226}, cluster_service_master="0.0.0.0:0") [2024-08-09 16:17:30.479592] WDIAG [STORAGE.TRANS] generate_min_weak_read_version (ob_weak_read_util.cpp:83) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=5][errcode=-4023] get gts cache error(ret=-4023, tenant_id=1) [2024-08-09 16:17:30.479585] WDIAG [SHARE.LOCATION] batch_process_tasks (ob_ls_location_service.cpp:549) [32514][SysLocAsyncUp0][T0][YB420A92023B-00061F37C607461A-0-0] [lt=48][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-08-09 16:17:30.479598] WDIAG [STORAGE.TRANS] generate_server_version (ob_tenant_weak_read_service.cpp:317) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=6][errcode=-4023] generate min weak read version error(ret=-4023, tenant_id=1) [2024-08-09 16:17:30.479601] WDIAG [STORAGE.TRANS] generate_tenant_weak_read_timestamp_ (ob_tenant_weak_read_service.cpp:597) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=3][errcode=-4023] generate server version for tenant fail(ret=-4023, ret="OB_EAGAIN", tenant_id=1, index=0x2aabd3de5f10, server_version_epoch_tstamp_=1723191450479585) [2024-08-09 16:17:30.480883] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023D-00061F37C6C806B0-0-0] [lt=6][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.482154] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023D-00061F37C6C806B0-0-0] [lt=32][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.485376] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=8][errcode=-4122] check_blacklist failed(addr="ip3:2882") [2024-08-09 16:17:30.485390] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=12][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, arg={addr:"ip3:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.485398] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=8][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip3:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.485613] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=25][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.485769] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0][errcode=-4122] check_blacklist failed(addr="ip4:2882") [2024-08-09 16:17:30.485778] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=7][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, arg={addr:"ip4:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.485782] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=4][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip4:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.485795] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.485804] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=8][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.485807] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:140) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=3] leader doesn't exist, try use all_server_list(tmp_ret=-4018, tmp_ret="OB_ENTRY_NOT_EXIST", ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.485814] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:151) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=7] server_list is empty, do nothing(ret=0, ret="OB_SUCCESS", server_list=[]) [2024-08-09 16:17:30.485821] INFO [SHARE.LOCATION] batch_update_caches_ (ob_ls_location_service.cpp:944) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=4] [LS_LOCATION]ls location cache has changed(ret=0, ret="OB_SUCCESS", old_location={cache_key:{tenant_id:0, ls_id:{id:-1}, cluster_id:-1}, renew_time:0, replica_locations:[]}, new_location={cache_key:{tenant_id:1, ls_id:{id:1}, cluster_id:1723101456}, renew_time:1723191450485820, replica_locations:[]}) [2024-08-09 16:17:30.485830] INFO [SHARE.LOCATION] batch_renew_tablet_locations (ob_location_service.cpp:442) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=8] [TABLET_LOCATION] batch renew tablet locations finished(ret=0, ret="OB_SUCCESS", tenant_id=1, renew_type=0, is_nonblock=false, tablet_list=[{id:1}], ls_ids=[{id:1}], error_code=-4721) [2024-08-09 16:17:30.485841] WDIAG [SQL] do_close_plan (ob_result_set.cpp:825) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0][errcode=-4006] exec result is null(ret=-4006) [2024-08-09 16:17:30.486312] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=1][errcode=-4122] check_blacklist failed(addr="ip3:2882") [2024-08-09 16:17:30.486325] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=11][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, arg={addr:"ip3:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.486332] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=7][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip3:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.486376] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=9][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.486439] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=1][errcode=-4122] check_blacklist failed(addr="ip4:2882") [2024-08-09 16:17:30.486448] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=7][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, arg={addr:"ip4:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.486452] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=4][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip4:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.486465] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.486473] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=8][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.486477] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:140) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=4] leader doesn't exist, try use all_server_list(tmp_ret=-4018, tmp_ret="OB_ENTRY_NOT_EXIST", ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.486482] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:151) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=4] server_list is empty, do nothing(ret=0, ret="OB_SUCCESS", server_list=[]) [2024-08-09 16:17:30.486488] INFO [SHARE.LOCATION] batch_update_caches_ (ob_ls_location_service.cpp:944) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=3] [LS_LOCATION]ls location cache has changed(ret=0, ret="OB_SUCCESS", old_location={cache_key:{tenant_id:0, ls_id:{id:-1}, cluster_id:-1}, renew_time:0, replica_locations:[]}, new_location={cache_key:{tenant_id:1, ls_id:{id:1}, cluster_id:1723101456}, renew_time:1723191450486487, replica_locations:[]}) [2024-08-09 16:17:30.486513] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:92) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=1] will sleep(sleep_us=37439, remain_us=37439, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=62, v.err_=-4721, timeout_timestamp=1723191450523951) [2024-08-09 16:17:30.487095] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:584) [32554][TsMgr][T0][Y0-0000000000000000-0-0] [lt=0] gts nonblock renew success(ret=0, tenant_id=1, gts_local_cache={srr:[mts=0], gts:0, latest_srr:[mts=0]}) [2024-08-09 16:17:30.487111] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:179) [32554][TsMgr][T0][Y0-0000000000000000-0-0] [lt=0] refresh gts functor(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", gts_tenant_info={v:1}) [2024-08-09 16:17:30.488383] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [32459][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=13] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=3) [2024-08-09 16:17:30.513229] INFO [SERVER] prepare_alloc_queue (ob_dl_queue.cpp:52) [32749][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=16] Construct Queue Num(construct_num=0, get_push_idx()=12, get_cur_idx()=4, get_pop_idx()=4) [2024-08-09 16:17:30.518078] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C6B6C7F9-0-0] [lt=8][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.518950] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C6B6C7F9-0-0] [lt=12][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.524029] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=7][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1723191450523951, ctx_timeout_ts=1723191450523951, worker_timeout_ts=1723191450523951, default_timeout=1000000) [2024-08-09 16:17:30.524046] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=16][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-08-09 16:17:30.524051] WDIAG [SHARE.LOCATION] batch_renew_tablet_locations (ob_location_service.cpp:429) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=5][errcode=-4012] batch renew cache failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1, tablet_list=[{id:1}], ls_ids=[{id:1}]) [2024-08-09 16:17:30.524058] INFO [SHARE.LOCATION] batch_renew_tablet_locations (ob_location_service.cpp:442) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=7] [TABLET_LOCATION] batch renew tablet locations finished(ret=-4012, ret="OB_TIMEOUT", tenant_id=1, renew_type=0, is_nonblock=false, tablet_list=[{id:1}], ls_ids=[{id:1}], error_code=-4721) [2024-08-09 16:17:30.524069] WDIAG [SQL.DAS] force_refresh_location_cache (ob_das_location_router.cpp:1212) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=9][errcode=-4012] batch renew tablet locations failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1, err_no=-4721, is_nonblock=false, failed_list=[{id:1}]) [2024-08-09 16:17:30.524081] WDIAG [SQL] do_close_plan (ob_result_set.cpp:825) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=1][errcode=-4006] exec result is null(ret=-4006) [2024-08-09 16:17:30.524522] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1723191450523951, ctx_timeout_ts=1723191450523951, worker_timeout_ts=1723191450523951, default_timeout=1000000) [2024-08-09 16:17:30.524538] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=15][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-08-09 16:17:30.524544] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=7][errcode=-4012] batch renew ls locations failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1723101456, tenant_id=1, ls_ids=[{id:1}]) [2024-08-09 16:17:30.524553] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=8][errcode=-4012] renew location failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1723101456, tenant_id=1, ls_id={id:1}) [2024-08-09 16:17:30.524558] WDIAG [SHARE.LOCATION] get (ob_location_service.cpp:58) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=4][errcode=-4012] fail to get log stream location(ret=-4012, ret="OB_TIMEOUT", cluster_id=1723101456, tenant_id=1, ls_id={id:1}, expire_renew_time=9223372036854775807, is_cache_hit=false, location={cache_key:{tenant_id:0, ls_id:{id:-1}, cluster_id:-1}, renew_time:0, replica_locations:[]}) [2024-08-09 16:17:30.524567] WDIAG [SQL.DAS] block_renew_tablet_location (ob_das_location_router.cpp:1247) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=10][errcode=-4012] failed to get location(ls_id={id:1}, ret=-4012) [2024-08-09 16:17:30.524588] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:104) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=1] already timeout, do not need sleep(sleep_us=-636, remain_us=-636, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=63, timeout_timestamp=1723191450523951) [2024-08-09 16:17:30.524598] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=7][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1723191450523951, ctx_timeout_ts=1723191450523951, worker_timeout_ts=1723191450523951, default_timeout=1000000) [2024-08-09 16:17:30.524603] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=5][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-08-09 16:17:30.524606] WDIAG [SHARE.LOCATION] batch_renew_tablet_locations (ob_location_service.cpp:429) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=3][errcode=-4012] batch renew cache failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1, tablet_list=[{id:1}], ls_ids=[{id:1}]) [2024-08-09 16:17:30.524613] INFO [SHARE.LOCATION] batch_renew_tablet_locations (ob_location_service.cpp:442) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=7] [TABLET_LOCATION] batch renew tablet locations finished(ret=-4012, ret="OB_TIMEOUT", tenant_id=1, renew_type=0, is_nonblock=false, tablet_list=[{id:1}], ls_ids=[{id:1}], error_code=-4721) [2024-08-09 16:17:30.524621] WDIAG [SQL.DAS] force_refresh_location_cache (ob_das_location_router.cpp:1212) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=7][errcode=-4012] batch renew tablet locations failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1, err_no=-4721, is_nonblock=false, failed_list=[{id:1}]) [2024-08-09 16:17:30.524629] WDIAG [SQL] do_close_plan (ob_result_set.cpp:825) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0][errcode=-4006] exec result is null(ret=-4006) [2024-08-09 16:17:30.525050] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1723191450523951, ctx_timeout_ts=1723191450523951, worker_timeout_ts=1723191450523951, default_timeout=1000000) [2024-08-09 16:17:30.525066] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=16][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-08-09 16:17:30.525070] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=4][errcode=-4012] batch renew ls locations failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1723101456, tenant_id=1, ls_ids=[{id:1}]) [2024-08-09 16:17:30.525078] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=8][errcode=-4012] renew location failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1723101456, tenant_id=1, ls_id={id:1}) [2024-08-09 16:17:30.525083] WDIAG [SHARE.LOCATION] get (ob_location_service.cpp:58) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=5][errcode=-4012] fail to get log stream location(ret=-4012, ret="OB_TIMEOUT", cluster_id=1723101456, tenant_id=1, ls_id={id:1}, expire_renew_time=9223372036854775807, is_cache_hit=false, location={cache_key:{tenant_id:0, ls_id:{id:-1}, cluster_id:-1}, renew_time:0, replica_locations:[]}) [2024-08-09 16:17:30.525092] WDIAG [SQL.DAS] block_renew_tablet_location (ob_das_location_router.cpp:1247) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=9][errcode=-4012] failed to get location(ls_id={id:1}, ret=-4012) [2024-08-09 16:17:30.525107] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:968) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:64, local_retry_times:64, err_:-4721, err_:"OB_LS_LOCATION_NOT_EXIST", retry_type:0, client_ret:-4012}, need_retry=false) [2024-08-09 16:17:30.525117] WDIAG [SQL] do_close_plan (ob_result_set.cpp:825) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=9][errcode=-4006] exec result is null(ret=-4006) [2024-08-09 16:17:30.525129] INFO [SHARE.LOCATION] batch_renew_tablet_locations (ob_location_service.cpp:442) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0] [TABLET_LOCATION] batch renew tablet locations finished(ret=0, ret="OB_SUCCESS", tenant_id=1, renew_type=0, is_nonblock=true, tablet_list=[{id:1}], ls_ids=[], error_code=-4721) [2024-08-09 16:17:30.525138] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:927) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0][errcode=-4012] failed to close result(close_ret=-4721, ret=-4012) [2024-08-09 16:17:30.525147] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:957) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=5][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, record_ret=-4012, ret=-4012) [2024-08-09 16:17:30.525154] INFO [SERVER] process_final (ob_inner_sql_connection.cpp:743) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=0] slow inner sql(last_ret=-4012, sql={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, process_time=2000680) [2024-08-09 16:17:30.525161] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:993) [32703][T1_FreInfoReloa][T1][YB420A92023B-00061F37C6C5A9D1-0-0] [lt=6][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, aret=-4012, ret=-4012) [2024-08-09 16:17:30.525166] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1786) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-08-09 16:17:30.525170] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1054) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1) [2024-08-09 16:17:30.525176] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1726) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1723101456, tenant_id=1) [2024-08-09 16:17:30.525181] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] query failed(ret=-4012, conn=0x2aabdeffc060, start=1723191448524469, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-08-09 16:17:30.525189] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=7][errcode=-4012] read failed(ret=-4012) [2024-08-09 16:17:30.525193] WDIAG [SHARE] load (ob_core_table_proxy.cpp:436) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=2][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-08-09 16:17:30.525247] WDIAG [SHARE] load (ob_core_table_proxy.cpp:368) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=0][errcode=-4012] load failed(ret=-4012, for_update=false) [2024-08-09 16:17:30.525256] WDIAG [SHARE] get (ob_global_stat_proxy.cpp:442) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=8][errcode=-4012] core_table load failed(ret=-4012, ret="OB_TIMEOUT") [2024-08-09 16:17:30.525260] WDIAG [SHARE] get_snapshot_gc_scn (ob_global_stat_proxy.cpp:164) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] get failed(ret=-4012) [2024-08-09 16:17:30.525266] WDIAG [STORAGE] get_global_info (ob_tenant_freeze_info_mgr.cpp:811) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] fail to get global info(ret=-4012, tenant_id=1) [2024-08-09 16:17:30.525273] WDIAG [STORAGE] try_update_info (ob_tenant_freeze_info_mgr.cpp:954) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=6][errcode=-4012] failed to get global info(ret=-4012) [2024-08-09 16:17:30.525276] WDIAG [STORAGE] runTimerTask (ob_tenant_freeze_info_mgr.cpp:1008) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] fail to try update info(tmp_ret=-4012, tmp_ret="OB_TIMEOUT") [2024-08-09 16:17:30.525286] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1044) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=5] success to update min reserved snapshot(reserved_snapshot=0, duration=1800, snapshot_gc_ts_=0) [2024-08-09 16:17:30.525292] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1071) [32703][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=6] update reserved snapshot finished(cost_ts=8, reserved_snapshot=0) [2024-08-09 16:17:30.531755] INFO [STORAGE.BLKMGR] runTimerTask (ob_block_manager.cpp:1573) [32529][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=13] skip inspect bad block(last_check_time=1723174263571312, last_macro_idx=-1) [2024-08-09 16:17:30.540117] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023D-00061F37C6C806B0-0-0] [lt=8][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.541217] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023D-00061F37C6C806B0-0-0] [lt=11][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.550420] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=8][errcode=-4122] check_blacklist failed(addr="ip3:2882") [2024-08-09 16:17:30.550436] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=14][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, arg={addr:"ip3:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.550444] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=7][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip3:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.550461] WDIAG [RPC] on_timeout (ob_async_rpc_proxy.h:83) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=17][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-08-09 16:17:30.550577] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023B-00061F37C6858EFA-0-0] [lt=27][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.550657] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=1][errcode=-4122] check_blacklist failed(addr="ip4:2882") [2024-08-09 16:17:30.550666] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=7][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, arg={addr:"ip4:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.550671] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip4:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.550682] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=0][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.550691] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=9][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.550696] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:140) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4] leader doesn't exist, try use all_server_list(tmp_ret=-4018, tmp_ret="OB_ENTRY_NOT_EXIST", ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.550703] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:151) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6] server_list is empty, do nothing(ret=0, ret="OB_SUCCESS", server_list=[]) [2024-08-09 16:17:30.550710] WDIAG [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:355) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6][errcode=-4638] no leader finded(ret=-4638, ret="OB_RS_NOT_MASTER", leader_exist=false, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.550718] INFO [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:366) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=7] [RS_MGR] new master rootserver found(rootservice="0.0.0.0:0", cluster_id=1723101456) [2024-08-09 16:17:30.550722] WDIAG [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:311) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4][errcode=-4638] failed to renew master rootserver(ret=-4638, ret="OB_RS_NOT_MASTER") [2024-08-09 16:17:30.550726] WDIAG [SERVER] register_self_busy_wait (ob_lease_state_mgr.cpp:169) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3][errcode=-4638] renew_master_rootserver failed(tmp_ret=-4638) [2024-08-09 16:17:30.550749] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=5][errcode=-4122] check_blacklist failed(addr="ip3:2882") [2024-08-09 16:17:30.550755] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=5][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, arg={addr:"ip3:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.550762] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=7][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip3:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.550797] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023B-00061F37C6858EFA-0-0] [lt=9][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.550876] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=0][errcode=-4122] check_blacklist failed(addr="ip4:2882") [2024-08-09 16:17:30.550883] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, arg={addr:"ip4:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.550887] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip4:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.550898] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=0][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.550904] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.550907] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:140) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3] leader doesn't exist, try use all_server_list(tmp_ret=-4018, tmp_ret="OB_ENTRY_NOT_EXIST", ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.550913] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:151) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6] server_list is empty, do nothing(ret=0, ret="OB_SUCCESS", server_list=[]) [2024-08-09 16:17:30.550919] INFO [SHARE.LOCATION] batch_update_caches_ (ob_ls_location_service.cpp:944) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3] [LS_LOCATION]ls location cache has changed(ret=0, ret="OB_SUCCESS", old_location={cache_key:{tenant_id:0, ls_id:{id:-1}, cluster_id:-1}, renew_time:0, replica_locations:[]}, new_location={cache_key:{tenant_id:1, ls_id:{id:1}, cluster_id:1723101456}, renew_time:1723191450550918, replica_locations:[]}) [2024-08-09 16:17:30.550930] WDIAG [SERVER] register_self_busy_wait (ob_lease_state_mgr.cpp:171) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=0][errcode=-4638] fail to refresh core partition(tmp_ret=-4721) [2024-08-09 16:17:30.551099] WDIAG [RPC] send (ob_poc_rpc_proxy.h:170) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3][errcode=-4638] execute rpc fail(addr="ip2:2882", pcode=258, ret=-4638, timeout=2000000) [2024-08-09 16:17:30.551108] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:246) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=7][errcode=-4638] [2024-08-09 16:17:30.551129] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3][errcode=-4122] check_blacklist failed(addr="ip3:2882") [2024-08-09 16:17:30.551135] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=5][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, arg={addr:"ip3:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551141] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip3:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551191] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023B-00061F37C6858EFA-0-0] [lt=8][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.551249] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=0][errcode=-4122] check_blacklist failed(addr="ip4:2882") [2024-08-09 16:17:30.551255] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=5][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, arg={addr:"ip4:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551259] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip4:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551269] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=0][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.551275] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.551278] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:140) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3] leader doesn't exist, try use all_server_list(tmp_ret=-4018, tmp_ret="OB_ENTRY_NOT_EXIST", ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.551282] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:151) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4] server_list is empty, do nothing(ret=0, ret="OB_SUCCESS", server_list=[]) [2024-08-09 16:17:30.551286] WDIAG [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:355) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3][errcode=-4638] no leader finded(ret=-4638, ret="OB_RS_NOT_MASTER", leader_exist=false, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.551292] INFO [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:366) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6] [RS_MGR] new master rootserver found(rootservice="0.0.0.0:0", cluster_id=1723101456) [2024-08-09 16:17:30.551295] WDIAG [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:311) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3][errcode=-4638] failed to renew master rootserver(ret=-4638, ret="OB_RS_NOT_MASTER") [2024-08-09 16:17:30.551298] WDIAG [SHARE] rpc_call (ob_common_rpc_proxy.h:419) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=2][errcode=-4638] renew_master_rootserver failed(ret=-4638, retry=0) [2024-08-09 16:17:30.551314] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3][errcode=-4122] check_blacklist failed(addr="ip3:2882") [2024-08-09 16:17:30.551319] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, arg={addr:"ip3:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551326] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip3:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551347] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023B-00061F37C6858EFA-0-0] [lt=8][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.551381] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=0][errcode=-4122] check_blacklist failed(addr="ip4:2882") [2024-08-09 16:17:30.551387] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=5][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, arg={addr:"ip4:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551391] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip4:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551399] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=0][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.551405] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.551408] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:140) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3] leader doesn't exist, try use all_server_list(tmp_ret=-4018, tmp_ret="OB_ENTRY_NOT_EXIST", ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.551412] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:151) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4] server_list is empty, do nothing(ret=0, ret="OB_SUCCESS", server_list=[]) [2024-08-09 16:17:30.551416] WDIAG [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:355) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4][errcode=-4638] no leader finded(ret=-4638, ret="OB_RS_NOT_MASTER", leader_exist=false, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.551420] INFO [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:366) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4] [RS_MGR] new master rootserver found(rootservice="0.0.0.0:0", cluster_id=1723101456) [2024-08-09 16:17:30.551423] WDIAG [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:311) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3][errcode=-4638] failed to renew master rootserver(ret=-4638, ret="OB_RS_NOT_MASTER") [2024-08-09 16:17:30.551425] WDIAG [SHARE] rpc_call (ob_common_rpc_proxy.h:419) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=2][errcode=-4638] renew_master_rootserver failed(ret=-4638, retry=1) [2024-08-09 16:17:30.551439] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3][errcode=-4122] check_blacklist failed(addr="ip3:2882") [2024-08-09 16:17:30.551452] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=5][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, arg={addr:"ip3:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551458] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip3:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551467] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023B-00061F37C6858EFA-0-0] [lt=7][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.551501] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=0][errcode=-4122] check_blacklist failed(addr="ip4:2882") [2024-08-09 16:17:30.551507] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=5][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, arg={addr:"ip4:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551511] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip4:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.551519] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=0][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.551525] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=6][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.551529] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:140) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4] leader doesn't exist, try use all_server_list(tmp_ret=-4018, tmp_ret="OB_ENTRY_NOT_EXIST", ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.551532] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:151) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3] server_list is empty, do nothing(ret=0, ret="OB_SUCCESS", server_list=[]) [2024-08-09 16:17:30.551536] WDIAG [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:355) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4][errcode=-4638] no leader finded(ret=-4638, ret="OB_RS_NOT_MASTER", leader_exist=false, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.551540] INFO [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:366) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=4] [RS_MGR] new master rootserver found(rootservice="0.0.0.0:0", cluster_id=1723101456) [2024-08-09 16:17:30.551543] WDIAG [SHARE] renew_master_rootserver (ob_rs_mgr.cpp:311) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3][errcode=-4638] failed to renew master rootserver(ret=-4638, ret="OB_RS_NOT_MASTER") [2024-08-09 16:17:30.551546] WDIAG [SHARE] rpc_call (ob_common_rpc_proxy.h:419) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=2][errcode=-4638] renew_master_rootserver failed(ret=-4638, retry=2) [2024-08-09 16:17:30.551549] WDIAG [SERVER] do_renew_lease (ob_lease_state_mgr.cpp:415) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=2][errcode=-4638] can't get lease from rs(rs_addr="ip2:2882", ret=-4638) [2024-08-09 16:17:30.551553] WDIAG [SERVER] register_self_busy_wait (ob_lease_state_mgr.cpp:160) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=3][errcode=-4638] fail to do_renew_lease(ret=-4638, ret="OB_RS_NOT_MASTER") [2024-08-09 16:17:30.551558] WDIAG [SERVER] register_self_busy_wait (ob_lease_state_mgr.cpp:165) [32426][observer][T0][YB420A92023B-00061F37C6858EFA-0-0] [lt=5][errcode=-4638] register failed, will try again(ret=-4638, ret="OB_RS_NOT_MASTER", retry latency=2) [2024-08-09 16:17:30.577721] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C6B6C7F9-0-0] [lt=7][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.578596] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C6B6C7F9-0-0] [lt=11][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.579620] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=3][errcode=-4076] tenant weak read service cluster heartbeat RPC fail(ret=-4076, rcode={code:-4076, msg:"post cluster heartbeat rpc failed, tenant_id=1", warnings:[]}, tenant_id_=1, dst="ip2:2882", cluster_service_tablet_id={id:226}) [2024-08-09 16:17:30.579653] WDIAG [STORAGE.TRANS] do_cluster_heartbeat_ (ob_tenant_weak_read_service.cpp:869) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=32][errcode=-4076] post cluster heartbeat rpc fail(ret=-4076, ret="OB_NEED_WAIT", tenant_id_=1, local_server_version={val:18446744073709551615, v:3}, valid_part_count=0, total_part_count=0, generate_timestamp=1723191450579609) [2024-08-09 16:17:30.579659] WDIAG [STORAGE.TRANS] do_cluster_heartbeat_ (ob_tenant_weak_read_service.cpp:879) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] tenant weak read service do cluster heartbeat fail(ret=-4076, ret="OB_NEED_WAIT", tenant_id_=1, last_post_cluster_heartbeat_tstamp_=1723191450479585, cluster_heartbeat_interval_=1000000, cluster_service_tablet_id={id:226}, cluster_service_master="0.0.0.0:0") [2024-08-09 16:17:30.579673] WDIAG [STORAGE.TRANS] generate_min_weak_read_version (ob_weak_read_util.cpp:83) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=8][errcode=-4023] get gts cache error(ret=-4023, tenant_id=1) [2024-08-09 16:17:30.579679] WDIAG [STORAGE.TRANS] generate_server_version (ob_tenant_weak_read_service.cpp:317) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=6][errcode=-4023] generate min weak read version error(ret=-4023, tenant_id=1) [2024-08-09 16:17:30.579682] WDIAG [STORAGE.TRANS] generate_tenant_weak_read_timestamp_ (ob_tenant_weak_read_service.cpp:597) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=3][errcode=-4023] generate server version for tenant fail(ret=-4023, ret="OB_EAGAIN", tenant_id=1, index=0x2aabd3de5f10, server_version_epoch_tstamp_=1723191450579667) [2024-08-09 16:17:30.600251] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023D-00061F37C6C806B0-0-0] [lt=7][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.601353] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023D-00061F37C6C806B0-0-0] [lt=13][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.602878] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [32756][T1_LSArchiveMgr][T1][YB420A92023B-00061F37C6158EFA-0-0] [lt=10] gc stale ls task succ [2024-08-09 16:17:30.638362] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C6B6C7F9-0-0] [lt=8][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.639279] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C6B6C7F9-0-0] [lt=13][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.661473] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023D-00061F37C6C806B0-0-0] [lt=8][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.662735] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1155) [32458][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-8345858048, global_cache_size=0, tenant_max_wash_size=0, tenant_min_wash_size=0, tenant_ids_=[500, 508, 1]) [2024-08-09 16:17:30.662859] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023D-00061F37C6C806B0-0-0] [lt=11][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.667573] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023D-00061F37C6A82ED9-0-0] [lt=12][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.679665] WDIAG [STORAGE.TRANS] generate_min_weak_read_version (ob_weak_read_util.cpp:83) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=6][errcode=-4023] get gts cache error(ret=-4023, tenant_id=1) [2024-08-09 16:17:30.679678] WDIAG [STORAGE.TRANS] generate_server_version (ob_tenant_weak_read_service.cpp:317) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=12][errcode=-4023] generate min weak read version error(ret=-4023, tenant_id=1) [2024-08-09 16:17:30.679683] WDIAG [STORAGE.TRANS] generate_tenant_weak_read_timestamp_ (ob_tenant_weak_read_service.cpp:597) [32750][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=4][errcode=-4023] generate server version for tenant fail(ret=-4023, ret="OB_EAGAIN", tenant_id=1, index=0x2aabd3de5f10, server_version_epoch_tstamp_=1723191450679656) [2024-08-09 16:17:30.688478] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [32459][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=32] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=31457, map_replace_skip_count_=2) [2024-08-09 16:17:30.699981] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C6B6C7F9-0-0] [lt=9][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.700992] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023C-00061F37C6B6C7F9-0-0] [lt=12][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.703765] INFO [SQL.RESV] check_table_exist_or_not (ob_dml_resolver.cpp:7588) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=8] table not exist(tenant_id=1, database_id=201001, table_name=__all_virtual_ls_meta_table, table_name.ptr()="data_size:27, data:5F5F616C6C5F7669727475616C5F6C735F6D6574615F7461626C65", ret=-5019) [2024-08-09 16:17:30.703787] WDIAG [SQL.RESV] resolve_table_relation_recursively (ob_dml_resolver.cpp:7546) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=20][errcode=-5019] synonym not exist(tenant_id=1, database_id=201001, table_name=__all_virtual_ls_meta_table, ret=-5019) [2024-08-09 16:17:30.703793] WDIAG [SQL.RESV] resolve_table_relation_factor_normal (ob_dml_resolver.cpp:7376) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=5][errcode=-5019] fail to resolve table relation recursively(tenant_id=1, ret=-5019, database_id=201001, database_id=201001, table_name=__all_virtual_ls_meta_table, db_name=oceanbase) [2024-08-09 16:17:30.703798] WDIAG [SQL.RESV] resolve_table_relation_factor (ob_dml_resolver.cpp:7219) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=4][errcode=-5019] resolve table relation factor failed(ret=-5019, table_name=__all_virtual_ls_meta_table) [2024-08-09 16:17:30.703803] WDIAG [SQL.RESV] inner_resolve_sys_view (ob_dml_resolver.cpp:2165) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=3][errcode=-5019] fail to resolve table(ret=-5019) [2024-08-09 16:17:30.703807] WDIAG [SQL.RESV] resolve_table_relation_factor_wrapper (ob_dml_resolver.cpp:2220) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=4][errcode=-5019] fail to resolve sys view(ret=-5019) [2024-08-09 16:17:30.703812] WDIAG resolve_basic_table_without_cte (ob_dml_resolver.cpp:2316) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=3][errcode=-5019] Table 'oceanbase.__all_virtual_ls_meta_table' doesn't exist [2024-08-09 16:17:30.703827] WDIAG [SQL.RESV] resolve_basic_table_with_cte (ob_dml_resolver.cpp:13351) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=3][errcode=-5019] resolve base or alias table factor failed(ret=-5019) [2024-08-09 16:17:30.703830] WDIAG [SQL.RESV] resolve_basic_table (ob_dml_resolver.cpp:13279) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=3][errcode=-5019] fail to resolve basic table with cte(ret=-5019) [2024-08-09 16:17:30.703834] WDIAG [SQL.RESV] resolve_table (ob_dml_resolver.cpp:2736) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=4][errcode=-5019] resolve basic table failed(ret=-5019) [2024-08-09 16:17:30.703838] WDIAG [SQL.RESV] resolve_joined_table_item (ob_dml_resolver.cpp:3379) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=3][errcode=-5019] resolve table failed(ret=-5019) [2024-08-09 16:17:30.703855] WDIAG [SQL.RESV] resolve_joined_table (ob_dml_resolver.cpp:2934) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=16][errcode=-5019] resolve joined table item failed(ret=-5019) [2024-08-09 16:17:30.703861] WDIAG [SQL.RESV] resolve_table (ob_dml_resolver.cpp:2788) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=5][errcode=-5019] resolve joined table failed(ret=-5019) [2024-08-09 16:17:30.703864] WDIAG [SQL.RESV] resolve_from_clause (ob_select_resolver.cpp:3698) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=2][errcode=-5019] fail to exec resolve_table(*table_node, table_item)(ret=-5019) [2024-08-09 16:17:30.703869] WDIAG [SQL.RESV] resolve_normal_query (ob_select_resolver.cpp:1110) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=4][errcode=-5019] fail to exec resolve_from_clause(parse_tree.children_[PARSE_SELECT_FROM])(ret=-5019) [2024-08-09 16:17:30.703872] WDIAG [SQL.RESV] resolve (ob_select_resolver.cpp:1314) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=3][errcode=-5019] resolve normal query failed(ret=-5019) [2024-08-09 16:17:30.703878] WDIAG [SQL.RESV] select_stmt_resolver_func (ob_resolver.cpp:188) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=5][errcode=-5019] execute stmt_resolver failed(ret=-5019, parse_tree.type_=3299) [2024-08-09 16:17:30.703887] WDIAG [SQL] generate_stmt (ob_sql.cpp:3033) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=6][errcode=-5019] failed to resolve(ret=-5019) [2024-08-09 16:17:30.703892] WDIAG [SQL] generate_physical_plan (ob_sql.cpp:3154) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=5][errcode=-5019] Failed to generate stmt(ret=-5019, result.get_exec_context().need_disconnect()=false) [2024-08-09 16:17:30.703897] WDIAG [SQL] handle_physical_plan (ob_sql.cpp:5029) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=4][errcode=-5019] Failed to generate plan(ret=-5019, result.get_exec_context().need_disconnect()=false) [2024-08-09 16:17:30.703903] WDIAG [SQL] handle_text_query (ob_sql.cpp:2725) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=6][errcode=-5019] fail to handle physical plan(ret=-5019) [2024-08-09 16:17:30.703907] WDIAG [SQL] stmt_query (ob_sql.cpp:229) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=4][errcode=-5019] fail to handle text query(stmt=select a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;, ret=-5019) [2024-08-09 16:17:30.703911] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:764) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=4][errcode=-5019] executor execute failed(ret=-5019) [2024-08-09 16:17:30.703917] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:922) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=5][errcode=-5019] execute failed(ret=-5019, tenant_id=1, executor={ObIExecutor:, sql:"select a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;"}, retry_cnt=0, local_sys_schema_version=1, local_tenant_schema_version=1) [2024-08-09 16:17:30.703926] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:968) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=8][errcode=-5019] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5019, err_:"OB_TABLE_NOT_EXIST", retry_type:0, client_ret:-5019}, need_retry=false) [2024-08-09 16:17:30.703935] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:220) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=8][errcode=-5019] result set close failed(ret=-5019) [2024-08-09 16:17:30.703938] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:200) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=3][errcode=-5019] result set close failed(ret=-5019) [2024-08-09 16:17:30.703943] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:927) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=5][errcode=-5019] failed to close result(close_ret=-5019, ret=-5019) [2024-08-09 16:17:30.703948] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:957) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=2][errcode=-5019] failed to process record(executor={ObIExecutor:, sql:"select a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;"}, record_ret=-5019, ret=-5019) [2024-08-09 16:17:30.703954] WDIAG [SQL] move_to_sqlstat_cache (ob_sql_stat_record.cpp:352) [335][BlackListServic][T1][YB420A92023B-00061F37C675A55B-0-0] [lt=6][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-08-09 16:17:30.703961] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:993) [335][BlackListServic][T0][YB420A92023B-00061F37C675A55B-0-0] [lt=7][errcode=-5019] failed to process final(executor={ObIExecutor:, sql:"select a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;"}, aret=-5019, ret=-5019) [2024-08-09 16:17:30.703966] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1786) [335][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=4][errcode=-5019] execute sql failed(ret=-5019, tenant_id=1, sql=select a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;) [2024-08-09 16:17:30.703969] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1054) [335][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=3][errcode=-5019] retry_while_no_tenant_resource failed(ret=-5019, tenant_id=1) [2024-08-09 16:17:30.703975] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1726) [335][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=5][errcode=-5019] execute_read failed(ret=-5019, cluster_id=1723101456, tenant_id=1) [2024-08-09 16:17:30.703978] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [335][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=3][errcode=-5019] query failed(ret=-5019, conn=0x2aabdeffc6e0, start=1723191450703611, sql=select a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;) [2024-08-09 16:17:30.703985] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:111) [335][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=7][errcode=-5019] read failed(ret=-5019) [2024-08-09 16:17:30.703989] WDIAG [STORAGE.TRANS] do_thread_task_ (ob_black_list.cpp:222) [335][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=3][errcode=-5019] fail to execute sql(ret=-5019, ret="OB_TABLE_NOT_EXIST", sql=select a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;) [2024-08-09 16:17:30.704025] INFO [STORAGE.TRANS] run1 (ob_black_list.cpp:194) [335][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=5] ls blacklist refresh finish(cost_time=996) [2024-08-09 16:17:30.718933] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:1062) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=3][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-08-09 16:17:30.718979] WDIAG [RPC] check_blacklist (ob_poc_rpc_proxy.cpp:272) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=17][errcode=-4122] address in blacklist(ret=-4122, addr="ip3:2882") [2024-08-09 16:17:30.718990] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=10][errcode=-4122] check_blacklist failed(addr="ip3:2882") [2024-08-09 16:17:30.718995] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, arg={addr:"ip3:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.719008] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=12][errcode=-4122] call rpc func failed(server="ip3:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip3:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.719013] WDIAG [RPC] on_timeout (ob_async_rpc_proxy.h:83) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=5][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-08-09 16:17:30.719034] WDIAG [SERVER] fill_ls_replica (ob_service.cpp:2761) [340][T1_L0_G9][T1][YB420A92023B-00061F37C695D205-0-0] [lt=26][errcode=-4719] get ls handle failed(ret=-4719, ret="OB_LS_NOT_EXIST") [2024-08-09 16:17:30.719146] WDIAG [RPC] post (ob_poc_rpc_proxy.h:229) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=0][errcode=-4122] check_blacklist failed(addr="ip4:2882") [2024-08-09 16:17:30.719156] WDIAG [RPC] call_rpc (ob_async_rpc_proxy.h:356) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=8][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, arg={addr:"ip4:2882", cluster_id:1723101456}, cluster_id=1723101456, tenant_id=1, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.719161] WDIAG [RPC] call (ob_async_rpc_proxy.h:290) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=5][errcode=-4122] call rpc func failed(server="ip4:2882", timeout=2000000, cluster_id=1723101456, tenant_id=1, arg={addr:"ip4:2882", cluster_id:1723101456}, group_id=9, ret=-4122, ret="OB_RPC_POST_ERROR") [2024-08-09 16:17:30.719175] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=0][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.719186] WDIAG [SHARE.PT] find_leader (ob_ls_info.cpp:847) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=11][errcode=-4018] fail to get leader replica(ret=-4018, ret="OB_ENTRY_NOT_EXIST", *this={tenant_id:1, ls_id:{id:1}, replicas:[]}, replica count=0) [2024-08-09 16:17:30.719191] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:140) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4] leader doesn't exist, try use all_server_list(tmp_ret=-4018, tmp_ret="OB_ENTRY_NOT_EXIST", ls_info={tenant_id:1, ls_id:{id:1}, replicas:[]}) [2024-08-09 16:17:30.719198] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:151) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=6] server_list is empty, do nothing(ret=0, ret="OB_SUCCESS", server_list=[]) [2024-08-09 16:17:30.719887] INFO [SQL.RESV] check_table_exist_or_not (ob_dml_resolver.cpp:7588) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=1] table not exist(tenant_id=1, database_id=201001, table_name=__all_ls_meta_table, table_name.ptr()="data_size:19, data:5F5F616C6C5F6C735F6D6574615F7461626C65", ret=-5019) [2024-08-09 16:17:30.719903] WDIAG [SQL.RESV] resolve_table_relation_recursively (ob_dml_resolver.cpp:7546) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=15][errcode=-5019] synonym not exist(tenant_id=1, database_id=201001, table_name=__all_ls_meta_table, ret=-5019) [2024-08-09 16:17:30.719909] WDIAG [SQL.RESV] resolve_table_relation_factor_normal (ob_dml_resolver.cpp:7376) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=5][errcode=-5019] fail to resolve table relation recursively(tenant_id=1, ret=-5019, database_id=201001, database_id=201001, table_name=__all_ls_meta_table, db_name=oceanbase) [2024-08-09 16:17:30.719915] WDIAG [SQL.RESV] resolve_table_relation_factor (ob_dml_resolver.cpp:7219) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=5][errcode=-5019] resolve table relation factor failed(ret=-5019, table_name=__all_ls_meta_table) [2024-08-09 16:17:30.719925] WDIAG [SQL.RESV] inner_resolve_sys_view (ob_dml_resolver.cpp:2165) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=9][errcode=-5019] fail to resolve table(ret=-5019) [2024-08-09 16:17:30.719928] WDIAG [SQL.RESV] resolve_table_relation_factor_wrapper (ob_dml_resolver.cpp:2220) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=3][errcode=-5019] fail to resolve sys view(ret=-5019) [2024-08-09 16:17:30.719934] WDIAG resolve_basic_table_without_cte (ob_dml_resolver.cpp:2316) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-5019] Table 'oceanbase.__all_ls_meta_table' doesn't exist [2024-08-09 16:17:30.719937] WDIAG [SQL.RESV] resolve_basic_table_with_cte (ob_dml_resolver.cpp:13351) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=3][errcode=-5019] resolve base or alias table factor failed(ret=-5019) [2024-08-09 16:17:30.719941] WDIAG [SQL.RESV] resolve_basic_table (ob_dml_resolver.cpp:13279) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=3][errcode=-5019] fail to resolve basic table with cte(ret=-5019) [2024-08-09 16:17:30.719945] WDIAG [SQL.RESV] resolve_table (ob_dml_resolver.cpp:2736) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=5][errcode=-5019] resolve basic table failed(ret=-5019) [2024-08-09 16:17:30.719949] WDIAG [SQL.RESV] resolve_from_clause (ob_select_resolver.cpp:3698) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-5019] fail to exec resolve_table(*table_node, table_item)(ret=-5019) [2024-08-09 16:17:30.719957] WDIAG [SQL.RESV] resolve_normal_query (ob_select_resolver.cpp:1110) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=8][errcode=-5019] fail to exec resolve_from_clause(parse_tree.children_[PARSE_SELECT_FROM])(ret=-5019) [2024-08-09 16:17:30.719960] WDIAG [SQL.RESV] resolve (ob_select_resolver.cpp:1314) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=3][errcode=-5019] resolve normal query failed(ret=-5019) [2024-08-09 16:17:30.719964] WDIAG [SQL.RESV] select_stmt_resolver_func (ob_resolver.cpp:188) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-5019] execute stmt_resolver failed(ret=-5019, parse_tree.type_=3299) [2024-08-09 16:17:30.719976] WDIAG [SQL] generate_stmt (ob_sql.cpp:3033) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=8][errcode=-5019] failed to resolve(ret=-5019) [2024-08-09 16:17:30.719981] WDIAG [SQL] generate_physical_plan (ob_sql.cpp:3154) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-5019] Failed to generate stmt(ret=-5019, result.get_exec_context().need_disconnect()=false) [2024-08-09 16:17:30.719989] WDIAG [SQL] handle_physical_plan (ob_sql.cpp:5029) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=7][errcode=-5019] Failed to generate plan(ret=-5019, result.get_exec_context().need_disconnect()=false) [2024-08-09 16:17:30.719994] WDIAG [SQL] handle_text_query (ob_sql.cpp:2725) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-5019] fail to handle physical plan(ret=-5019) [2024-08-09 16:17:30.719998] WDIAG [SQL] stmt_query (ob_sql.cpp:229) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-5019] fail to handle text query(stmt=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1 ORDER BY tenant_id, ls_id, svr_ip, svr_port, ret=-5019) [2024-08-09 16:17:30.720006] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:764) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=7][errcode=-5019] executor execute failed(ret=-5019) [2024-08-09 16:17:30.720011] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:922) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-5019] execute failed(ret=-5019, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1, local_tenant_schema_version=1) [2024-08-09 16:17:30.720033] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:968) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=19][errcode=-5019] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5019, err_:"OB_TABLE_NOT_EXIST", retry_type:0, client_ret:-5019}, need_retry=false) [2024-08-09 16:17:30.720042] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:220) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=8][errcode=-5019] result set close failed(ret=-5019) [2024-08-09 16:17:30.720046] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:200) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-5019] result set close failed(ret=-5019) [2024-08-09 16:17:30.720049] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:927) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=3][errcode=-5019] failed to close result(close_ret=-5019, ret=-5019) [2024-08-09 16:17:30.720058] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:957) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-5019] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5019, ret=-5019) [2024-08-09 16:17:30.720083] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:993) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=1][errcode=-5019] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5019, ret=-5019) [2024-08-09 16:17:30.720091] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1786) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=8][errcode=-5019] execute sql failed(ret=-5019, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-08-09 16:17:30.720108] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1054) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=17][errcode=-5019] retry_while_no_tenant_resource failed(ret=-5019, tenant_id=1) [2024-08-09 16:17:30.720115] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1726) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=7][errcode=-5019] execute_read failed(ret=-5019, cluster_id=1723101456, tenant_id=1) [2024-08-09 16:17:30.720120] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=4][errcode=-5019] query failed(ret=-5019, conn=0x2aabdeffc3a0, start=1723191450719780, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-08-09 16:17:30.720126] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=6][errcode=-5019] read failed(ret=-5019) [2024-08-09 16:17:30.720130] WDIAG [SHARE.PT] get_by_tenant (ob_persistent_ls_table.cpp:639) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=3][errcode=-5019] execute sql failed(ret=-5019, ret="OB_TABLE_NOT_EXIST", tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-08-09 16:17:30.720169] WDIAG [SHARE.PT] get_by_tenant (ob_ls_table_operator.cpp:252) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=7][errcode=-5019] get all ls info by persistent_ls_ failed(ret=-5019, ret="OB_TABLE_NOT_EXIST", tenant_id=1) [2024-08-09 16:17:30.720177] WDIAG [SHARE] inner_open_ (ob_ls_table_iterator.cpp:104) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=8][errcode=-5019] fail to get ls infos by tenant(ret=-5019, ret="OB_TABLE_NOT_EXIST", tenant_id=1, inner_table_only=true) [2024-08-09 16:17:30.720182] WDIAG [SHARE] next (ob_ls_table_iterator.cpp:71) [32738][T1_LSMetaCh][T1][YB420A92023B-00061F37C695D205-0-0] [lt=5][errcode=-5019] fail to open iterator(ret=-5019, ret="OB_TABLE_NOT_EXIST")