1 [2024-06-06 11:28:14.658362] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502414-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602414-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702414-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 2 [2024-06-06 11:28:14.715779] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 3 [2024-06-06 11:28:14.716336] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 4 [2024-06-06 11:28:14.716354] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=23] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 5 [2024-06-06 11:28:14.717642] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 6 [2024-06-06 11:28:14.758463] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 7 [2024-06-06 11:28:14.758510] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 8 [2024-06-06 11:28:14.779670] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502415-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 9 [2024-06-06 11:28:14.779711] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502415-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 10 [2024-06-06 11:28:14.788635] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=24] PNIO [ratelimit] time: 1717644494788633, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007614, add_bytes: 0 11 [2024-06-06 11:28:14.795378] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=17] PNIO [ratelimit] time: 1717644494795376, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007618, add_bytes: 0 12 [2024-06-06 11:28:14.798594] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602415-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 13 [2024-06-06 11:28:14.798627] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602415-0-0] [lt=30] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 14 [2024-06-06 11:28:14.800539] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702415-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 15 [2024-06-06 11:28:14.800576] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702415-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 16 [2024-06-06 11:28:14.858641] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 17 [2024-06-06 11:28:14.858707] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=63][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502415-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602415-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702415-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 18 [2024-06-06 11:28:14.858786] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=78][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 19 [2024-06-06 11:28:14.858794] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502415-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602415-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702415-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 20 [2024-06-06 11:28:14.937646] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=15] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 21 [2024-06-06 11:28:14.937905] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=26] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 22 [2024-06-06 11:28:14.938113] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=12] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 23 [2024-06-06 11:28:14.938702] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 24 [2024-06-06 11:28:14.958901] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 25 [2024-06-06 11:28:14.958944] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 26 [2024-06-06 11:28:14.980665] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=3/0, request done=6928/0, request doing=0/0) 27 [2024-06-06 11:28:14.980773] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=6928/0, request doing=0/0) 28 [2024-06-06 11:28:14.980867] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=24] [RPC EASY STAT](log_str=conn count=3/0, request done=6928/0, request doing=0/0) 29 [2024-06-06 11:28:14.981082] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=3/0, request done=6927/0, request doing=1/0) 30 [2024-06-06 11:28:14.982343] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=16] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 31 [2024-06-06 11:28:14.982346] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=11] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 32 [2024-06-06 11:28:14.982347] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=9] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 33 [2024-06-06 11:28:14.983374] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 34 [2024-06-06 11:28:15.003194] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502416-0-0] [lt=21] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 35 [2024-06-06 11:28:15.003233] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502416-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 36 [2024-06-06 11:28:15.026071] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602416-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 37 [2024-06-06 11:28:15.026130] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602416-0-0] [lt=53] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 38 [2024-06-06 11:28:15.027816] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702416-0-0] [lt=19] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 39 [2024-06-06 11:28:15.027846] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702416-0-0] [lt=28] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 40 [2024-06-06 11:28:15.059053] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=27][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 41 [2024-06-06 11:28:15.059084] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502416-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602416-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702416-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 42 [2024-06-06 11:28:15.059125] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=37][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 43 [2024-06-06 11:28:15.059131] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502416-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602416-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702416-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 44 [2024-06-06 11:28:15.159233] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 45 [2024-06-06 11:28:15.159280] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 46 [2024-06-06 11:28:15.200840] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502417-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 47 [2024-06-06 11:28:15.200885] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502417-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 48 [2024-06-06 11:28:15.222033] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602417-0-0] [lt=28] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 49 [2024-06-06 11:28:15.222064] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602417-0-0] [lt=29] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 50 [2024-06-06 11:28:15.223917] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702417-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 51 [2024-06-06 11:28:15.223946] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702417-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 52 [2024-06-06 11:28:15.259355] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 53 [2024-06-06 11:28:15.259379] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502417-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602417-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702417-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 54 [2024-06-06 11:28:15.259416] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 55 [2024-06-06 11:28:15.259423] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502417-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602417-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702417-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 56 [2024-06-06 11:28:15.359523] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 57 [2024-06-06 11:28:15.359575] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=35] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 58 [2024-06-06 11:28:15.404758] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502418-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 59 [2024-06-06 11:28:15.404807] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502418-0-0] [lt=45] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 60 [2024-06-06 11:28:15.423727] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602418-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 61 [2024-06-06 11:28:15.423777] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602418-0-0] [lt=46] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 62 [2024-06-06 11:28:15.425749] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702418-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 63 [2024-06-06 11:28:15.425783] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702418-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 64 [2024-06-06 11:28:15.459672] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 65 [2024-06-06 11:28:15.459701] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502418-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602418-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702418-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 66 [2024-06-06 11:28:15.459735] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 67 [2024-06-06 11:28:15.459742] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502418-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602418-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702418-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 68 [2024-06-06 11:28:15.559841] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 69 [2024-06-06 11:28:15.559887] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 70 [2024-06-06 11:28:15.630144] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502419-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 71 [2024-06-06 11:28:15.630191] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502419-0-0] [lt=44] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 72 [2024-06-06 11:28:15.649385] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602419-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 73 [2024-06-06 11:28:15.649429] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602419-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 74 [2024-06-06 11:28:15.651465] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702419-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 75 [2024-06-06 11:28:15.651491] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702419-0-0] [lt=24] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 76 [2024-06-06 11:28:15.659986] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 77 [2024-06-06 11:28:15.660007] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=19][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502419-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602419-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702419-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 78 [2024-06-06 11:28:15.660036] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 79 [2024-06-06 11:28:15.660045] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=8][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502419-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602419-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702419-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 80 [2024-06-06 11:28:15.716446] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 81 [2024-06-06 11:28:15.716824] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 82 [2024-06-06 11:28:15.717344] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 83 [2024-06-06 11:28:15.717346] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 84 [2024-06-06 11:28:15.760152] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 85 [2024-06-06 11:28:15.760198] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=29] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 86 [2024-06-06 11:28:15.784182] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050241A-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 87 [2024-06-06 11:28:15.784228] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050241A-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 88 [2024-06-06 11:28:15.796253] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=20] PNIO [ratelimit] time: 1717644495796251, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007618, add_bytes: 0 89 [2024-06-06 11:28:15.802995] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=17] PNIO [ratelimit] time: 1717644495802994, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007618, add_bytes: 0 90 [2024-06-06 11:28:15.805371] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060241A-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 91 [2024-06-06 11:28:15.805406] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060241A-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 92 [2024-06-06 11:28:15.807521] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070241A-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 93 [2024-06-06 11:28:15.807549] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070241A-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 94 [2024-06-06 11:28:15.860296] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 95 [2024-06-06 11:28:15.860326] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241A-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 96 [2024-06-06 11:28:15.860369] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=43][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 97 [2024-06-06 11:28:15.860376] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241A-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 98 [2024-06-06 11:28:15.937689] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=24] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 99 [2024-06-06 11:28:15.938354] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 100 [2024-06-06 11:28:15.938921] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 101 [2024-06-06 11:28:15.939130] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=11] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 102 [2024-06-06 11:28:15.960474] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 103 [2024-06-06 11:28:15.960511] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 104 [2024-06-06 11:28:15.980576] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=3/0, request done=6932/0, request doing=0/0) 105 [2024-06-06 11:28:15.980659] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=22] [RPC EASY STAT](log_str=conn count=3/0, request done=6931/0, request doing=1/0) 106 [2024-06-06 11:28:15.980728] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=3/0, request done=6932/0, request doing=0/0) 107 [2024-06-06 11:28:15.981358] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=3/0, request done=6931/0, request doing=0/0) 108 [2024-06-06 11:28:15.982015] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050241B-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 109 [2024-06-06 11:28:15.982067] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050241B-0-0] [lt=47] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 110 [2024-06-06 11:28:15.982348] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=13] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 111 [2024-06-06 11:28:15.982358] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=16] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 112 [2024-06-06 11:28:15.983360] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 113 [2024-06-06 11:28:15.983394] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=14] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 114 [2024-06-06 11:28:16.000858] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060241B-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 115 [2024-06-06 11:28:16.000919] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060241B-0-0] [lt=57] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 116 [2024-06-06 11:28:16.002675] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070241B-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 117 [2024-06-06 11:28:16.002715] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070241B-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 118 [2024-06-06 11:28:16.060614] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 119 [2024-06-06 11:28:16.060644] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=29][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241B-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 120 [2024-06-06 11:28:16.060689] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=44][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 121 [2024-06-06 11:28:16.060695] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=4][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241B-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 122 [2024-06-06 11:28:16.145015] INFO [TLOG.FETCHER] print_stat (ob_log_fetch_stream_container_mgr.cpp:181) [51002][][T0][Y0-0000000000000000-0-0] [lt=25] [STAT] [FS_CONTAINER_POOL] USED=1 FREE=0 FIXED=16 DYNAMIC=0 123 [2024-06-06 11:28:16.145039] INFO [TLOG.FETCHER] print_stat (ob_log_fetch_stream_pool.cpp:76) [51002][][T0][Y0-0000000000000000-0-0] [lt=23] [STAT] [FETCH_STREAM_POOL] USED=0 FREE=0 FIXED=16 DYNAMIC=0 124 [2024-06-06 11:28:16.145044] INFO [TLOG.FETCHER] print_stat (ob_log_fetch_log_rpc.cpp:1400) [51002][][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [RPC_RESULT_POOL] USED=0 FREE=0 FIXED=16 DYNAMIC=0 125 [2024-06-06 11:28:16.145075] INFO [TLOG.FETCHER] operator() (ob_log_fetch_stream_container_mgr.cpp:230) [51002][][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [FETCH_STREAM] TENANT=1020, TRAFFIC=0.00B/sec 126 [2024-06-06 11:28:16.145105] INFO [TLOG.FETCHER] print_fetcher_stat_ (ob_log_fetcher.cpp:840) [51002][][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [FETCHER](upper_limit="[2024-06-04 16:49:13.886536]", dml_progress_limit_sec=30, fetcher_delay="[42:39:32.258558]") 127 [2024-06-06 11:28:16.145181] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:429) [51002][][T0][Y0-0000000000000000-0-0] [lt=7] [STAT] slow fetch progress start. ls_count=1/1 128 [2024-06-06 11:28:16.145196] INFO [TLOG.FETCHER] print_fetch_progress (ob_log_ls_fetch_ctx.cpp:1454) [51002][][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] slow fetch progress idx=0/1 tls_id={tenant_id:1020, ls_id:{id:1}} mod=IDLE_POOL discarded=0 delay=[42:39:32.258645] tps=0.00 progress={next_lsn:{lsn:18446744073709551615}, log_progress:"[2024-06-04 16:48:43.886536]", log_touch_tstamp:"[2024-06-06 10:57:10.051911]"} 129 [2024-06-06 11:28:16.145206] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:436) [51002][][T0][Y0-0000000000000000-0-0] [lt=10] [STAT] slow fetch progress end 130 [2024-06-06 11:28:16.145211] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:441) [51002][][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] slow dispatch progress start. ls_count=1/1 131 [2024-06-06 11:28:16.145218] INFO [TLOG.FETCHER] print_dispatch_progress (ob_log_ls_fetch_ctx.cpp:1476) [51002][][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] slow dispatch progress idx=0/1 tls_id={tenant_id:1020, ls_id:{id:1}} delay=[42:39:32.258645] pending_task(queue/total)=0/0 dispatch_progress=[2024-06-04 16:48:43.886536] last_dispatch_log_lsn=18446744073709551615 next_task=INVALID next_trans(log_lsn=18446744073709551615,committed=0,ready_to_commit=0,global_version=[1970-01-01 08:00:00.000000]) checkpoint=[1970-01-01 08:00:00.000000] 132 [2024-06-06 11:28:16.145224] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:448) [51002][][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] slow dispatch progress end 133 [2024-06-06 11:28:16.160795] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 134 [2024-06-06 11:28:16.160843] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 135 [2024-06-06 11:28:16.188993] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050241C-0-0] [lt=45] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 136 [2024-06-06 11:28:16.189030] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050241C-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 137 [2024-06-06 11:28:16.208089] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060241C-0-0] [lt=25] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 138 [2024-06-06 11:28:16.208141] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060241C-0-0] [lt=49] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 139 [2024-06-06 11:28:16.218074] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070241C-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 140 [2024-06-06 11:28:16.218116] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070241C-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 141 [2024-06-06 11:28:16.260939] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 142 [2024-06-06 11:28:16.260963] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241C-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 143 [2024-06-06 11:28:16.260998] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 144 [2024-06-06 11:28:16.261005] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241C-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 145 [2024-06-06 11:28:16.361118] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 146 [2024-06-06 11:28:16.361174] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=39] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 147 [2024-06-06 11:28:16.386065] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050241D-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 148 [2024-06-06 11:28:16.386109] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050241D-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 149 [2024-06-06 11:28:16.404810] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060241D-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 150 [2024-06-06 11:28:16.404866] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060241D-0-0] [lt=53] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 151 [2024-06-06 11:28:16.406720] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070241D-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 152 [2024-06-06 11:28:16.406747] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070241D-0-0] [lt=25] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 153 [2024-06-06 11:28:16.461274] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 154 [2024-06-06 11:28:16.461302] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241D-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 155 [2024-06-06 11:28:16.461348] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=45][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 156 [2024-06-06 11:28:16.461355] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241D-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 157 [2024-06-06 11:28:16.561460] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=27] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 158 [2024-06-06 11:28:16.561510] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 159 [2024-06-06 11:28:16.588560] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050241E-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 160 [2024-06-06 11:28:16.588603] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050241E-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 161 [2024-06-06 11:28:16.607175] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060241E-0-0] [lt=29] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 162 [2024-06-06 11:28:16.607206] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060241E-0-0] [lt=29] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 163 [2024-06-06 11:28:16.609422] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070241E-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 164 [2024-06-06 11:28:16.609448] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070241E-0-0] [lt=24] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 165 [2024-06-06 11:28:16.661606] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 166 [2024-06-06 11:28:16.661639] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241E-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 167 [2024-06-06 11:28:16.661675] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 168 [2024-06-06 11:28:16.661682] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241E-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 169 [2024-06-06 11:28:16.716736] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 170 [2024-06-06 11:28:16.716784] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 171 [2024-06-06 11:28:16.717348] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 172 [2024-06-06 11:28:16.717490] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=23] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 173 [2024-06-06 11:28:16.761783] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 174 [2024-06-06 11:28:16.761826] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 175 [2024-06-06 11:28:16.782358] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050241F-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 176 [2024-06-06 11:28:16.782391] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050241F-0-0] [lt=29] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 177 [2024-06-06 11:28:16.803650] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060241F-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 178 [2024-06-06 11:28:16.803680] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060241F-0-0] [lt=28] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 179 [2024-06-06 11:28:16.803867] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=25] PNIO [ratelimit] time: 1717644496803866, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007615, add_bytes: 0 180 [2024-06-06 11:28:16.810619] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=24] PNIO [ratelimit] time: 1717644496810608, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007614, add_bytes: 0 181 [2024-06-06 11:28:16.818405] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070241F-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 182 [2024-06-06 11:28:16.818442] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070241F-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 183 [2024-06-06 11:28:16.834509] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=25] [STAT] [ALL_ZONE] INDEX=0/3 ZONE=az1 REGION=BJ STORAGE_TYPE=LOCAL VERSION=371 184 [2024-06-06 11:28:16.834532] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=19] [STAT] [ALL_ZONE] INDEX=1/3 ZONE=az2 REGION=BJ STORAGE_TYPE=LOCAL VERSION=371 185 [2024-06-06 11:28:16.834539] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=6] [STAT] [ALL_ZONE] INDEX=2/3 ZONE=az4 REGION=BJ STORAGE_TYPE=LOCAL VERSION=371 186 [2024-06-06 11:28:16.834546] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [ALL_ZONE] INDEX=0/3 ZONE=az1 ZONE_TYPE=ReadWrite VERSION=371 187 [2024-06-06 11:28:16.834551] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [ALL_ZONE] INDEX=1/3 ZONE=az2 ZONE_TYPE=ReadWrite VERSION=371 188 [2024-06-06 11:28:16.834556] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_ZONE] INDEX=2/3 ZONE=az4 ZONE_TYPE=ReadWrite VERSION=371 189 [2024-06-06 11:28:16.834560] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:454) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_ZONE] COUNT=3 VERSION=371 190 [2024-06-06 11:28:16.834572] INFO [TLOG] purge_stale_zone_records_ (ob_log_all_svr_cache.cpp:660) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_ZONE] [PURGE] PURGE_COUNT=0 CUR_COUNT=3 VERSION=371 191 [2024-06-06 11:28:16.847636] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [ALL_SERVER_LIST] INDEX=0/15 SERVER_ID=4 SERVER="10.50.54.157:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=371 192 [2024-06-06 11:28:16.847663] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=25] [STAT] [ALL_SERVER_LIST] INDEX=1/15 SERVER_ID=6 SERVER="10.50.54.245:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=371 193 [2024-06-06 11:28:16.847670] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [ALL_SERVER_LIST] INDEX=2/15 SERVER_ID=5 SERVER="10.50.56.27:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=371 194 [2024-06-06 11:28:16.847674] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=3/15 SERVER_ID=2 SERVER="10.50.56.28:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=371 195 [2024-06-06 11:28:16.847679] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_SERVER_LIST] INDEX=4/15 SERVER_ID=1 SERVER="10.50.73.94:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=371 196 [2024-06-06 11:28:16.847683] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=5/15 SERVER_ID=9 SERVER="10.52.18.156:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=371 197 [2024-06-06 11:28:16.847688] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=6/15 SERVER_ID=8 SERVER="10.52.18.158:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=371 198 [2024-06-06 11:28:16.847692] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=7/15 SERVER_ID=7 SERVER="10.52.18.180:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=371 199 [2024-06-06 11:28:16.847697] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=8/15 SERVER_ID=13 SERVER="10.53.0.225:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=371 200 [2024-06-06 11:28:16.847701] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=9/15 SERVER_ID=15 SERVER="10.53.73.35:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=371 201 [2024-06-06 11:28:16.847706] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=10/15 SERVER_ID=14 SERVER="10.53.73.37:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=371 202 [2024-06-06 11:28:16.847710] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=11/15 SERVER_ID=10 SERVER="10.88.10.155:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=371 203 [2024-06-06 11:28:16.847714] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=12/15 SERVER_ID=3 SERVER="10.88.10.156:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=371 204 [2024-06-06 11:28:16.847719] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=13/15 SERVER_ID=11 SERVER="10.88.20.97:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=371 205 [2024-06-06 11:28:16.847723] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=14/15 SERVER_ID=12 SERVER="10.88.8.206:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=371 206 [2024-06-06 11:28:16.847726] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:525) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] COUNT=15 VERSION=371 207 [2024-06-06 11:28:16.847736] INFO [TLOG] purge_stale_records_ (ob_log_all_svr_cache.cpp:646) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_SERVER_LIST] [PURGE] PURGE_COUNT=0 CUR_COUNT=15 VERSION=371 208 [2024-06-06 11:28:16.861922] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 209 [2024-06-06 11:28:16.861947] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241F-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 210 [2024-06-06 11:28:16.861984] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 211 [2024-06-06 11:28:16.861990] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050241F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060241F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070241F-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 212 [2024-06-06 11:28:16.893683] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24432674440}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}]) 213 [2024-06-06 11:28:16.893710] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=26] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24432674440}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24432674440}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}]) 214 [2024-06-06 11:28:16.893729] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=19] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24432674440}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24432674440}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}, {server:"10.88.8.206:2882", ranges:[LSN:{24291926016, 24432674440}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}]) 215 [2024-06-06 11:28:16.893740] INFO [TLOG] get_ls_svr_list_ (ob_log_route_service.cpp:809) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=8] get_ls_svr_list_ done(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}, svr_list={next_svr_index:0, svr_num:3, svr_items:[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24432674440}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.88.8.206:2882", ranges:[LSN:{24291926016, 24432674440}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24432674440}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}]}, tg_id_=259) 216 [2024-06-06 11:28:16.937728] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=20] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 217 [2024-06-06 11:28:16.938739] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=19] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 218 [2024-06-06 11:28:16.938786] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=12] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 219 [2024-06-06 11:28:16.938944] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 220 [2024-06-06 11:28:16.962086] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 221 [2024-06-06 11:28:16.962126] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 222 [2024-06-06 11:28:16.980572] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=59] [RPC EASY STAT](log_str=conn count=3/0, request done=6936/0, request doing=0/0) 223 [2024-06-06 11:28:16.980833] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=3/0, request done=6935/0, request doing=0/0) 224 [2024-06-06 11:28:16.981254] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=3/0, request done=6935/0, request doing=1/0) 225 [2024-06-06 11:28:16.981558] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=33] [RPC EASY STAT](log_str=conn count=3/0, request done=6935/0, request doing=0/0) 226 [2024-06-06 11:28:16.981792] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=21] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 227 [2024-06-06 11:28:16.982389] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=27] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 228 [2024-06-06 11:28:16.982397] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=26] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 229 [2024-06-06 11:28:16.982654] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 230 [2024-06-06 11:28:17.036175] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502420-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 231 [2024-06-06 11:28:17.036228] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502420-0-0] [lt=50] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 232 [2024-06-06 11:28:17.080205] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602420-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 233 [2024-06-06 11:28:17.080252] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602420-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 234 [2024-06-06 11:28:17.084681] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702420-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 235 [2024-06-06 11:28:17.084713] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702420-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 236 [2024-06-06 11:28:17.145290] INFO [TLOG.PARSER] stat_ignored_tps (ob_log_cluster_id_filter.cpp:130) [51002][][T0][Y0-0000000000000000-0-0] [lt=2] [TPS_STAT] CLUSTER_ID_IGNORED_PART_TPS=0.000 237 [2024-06-06 11:28:17.162290] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 238 [2024-06-06 11:28:17.162319] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502420-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602420-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702420-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 239 [2024-06-06 11:28:17.162362] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=42][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 240 [2024-06-06 11:28:17.162368] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502420-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602420-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702420-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 241 [2024-06-06 11:28:17.262464] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 242 [2024-06-06 11:28:17.262517] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 243 [2024-06-06 11:28:17.293683] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502421-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 244 [2024-06-06 11:28:17.293718] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502421-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 245 [2024-06-06 11:28:17.315660] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602421-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 246 [2024-06-06 11:28:17.315696] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602421-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 247 [2024-06-06 11:28:17.320986] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702421-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 248 [2024-06-06 11:28:17.321012] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702421-0-0] [lt=24] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 249 [2024-06-06 11:28:17.362612] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 250 [2024-06-06 11:28:17.362640] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502421-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602421-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702421-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 251 [2024-06-06 11:28:17.362677] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 252 [2024-06-06 11:28:17.362683] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502421-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602421-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702421-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 253 [2024-06-06 11:28:17.462781] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 254 [2024-06-06 11:28:17.462830] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 255 [2024-06-06 11:28:17.504690] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502422-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 256 [2024-06-06 11:28:17.504737] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502422-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 257 [2024-06-06 11:28:17.524403] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602422-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 258 [2024-06-06 11:28:17.524442] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602422-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 259 [2024-06-06 11:28:17.526196] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702422-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 260 [2024-06-06 11:28:17.526222] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702422-0-0] [lt=25] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 261 [2024-06-06 11:28:17.562928] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 262 [2024-06-06 11:28:17.562955] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=25][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502422-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602422-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702422-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 263 [2024-06-06 11:28:17.562991] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 264 [2024-06-06 11:28:17.562998] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502422-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602422-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702422-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 265 [2024-06-06 11:28:17.663091] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 266 [2024-06-06 11:28:17.663139] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 267 [2024-06-06 11:28:17.702312] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502423-0-0] [lt=23] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 268 [2024-06-06 11:28:17.702362] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502423-0-0] [lt=46] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 269 [2024-06-06 11:28:17.716372] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 270 [2024-06-06 11:28:17.716422] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 271 [2024-06-06 11:28:17.716506] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 272 [2024-06-06 11:28:17.717518] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 273 [2024-06-06 11:28:17.721985] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602423-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 274 [2024-06-06 11:28:17.722011] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602423-0-0] [lt=24] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 275 [2024-06-06 11:28:17.723884] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702423-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 276 [2024-06-06 11:28:17.723916] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702423-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 277 [2024-06-06 11:28:17.763195] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 278 [2024-06-06 11:28:17.763224] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502423-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602423-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702423-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 279 [2024-06-06 11:28:17.763264] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=39][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 280 [2024-06-06 11:28:17.763270] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502423-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602423-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702423-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 281 [2024-06-06 11:28:17.811488] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=13] PNIO [ratelimit] time: 1717644497811484, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007618, add_bytes: 0 282 [2024-06-06 11:28:17.818230] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=16] PNIO [ratelimit] time: 1717644497818227, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007619, add_bytes: 0 283 [2024-06-06 11:28:17.863365] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 284 [2024-06-06 11:28:17.863412] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=29] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 285 [2024-06-06 11:28:17.896501] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502424-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 286 [2024-06-06 11:28:17.896553] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502424-0-0] [lt=48] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 287 [2024-06-06 11:28:17.917717] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602424-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 288 [2024-06-06 11:28:17.917762] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602424-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 289 [2024-06-06 11:28:17.928201] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702424-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 290 [2024-06-06 11:28:17.928237] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702424-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 291 [2024-06-06 11:28:17.937765] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=22] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 292 [2024-06-06 11:28:17.938355] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 293 [2024-06-06 11:28:17.938812] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=11] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 294 [2024-06-06 11:28:17.938836] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 295 [2024-06-06 11:28:17.963514] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=29][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 296 [2024-06-06 11:28:17.963537] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=22][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502424-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602424-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702424-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 297 [2024-06-06 11:28:17.963571] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 298 [2024-06-06 11:28:17.963577] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502424-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602424-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702424-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 299 [2024-06-06 11:28:17.980583] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=3/0, request done=6939/0, request doing=0/0) 300 [2024-06-06 11:28:17.980782] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=3/0, request done=6939/0, request doing=0/0) 301 [2024-06-06 11:28:17.981152] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=3/0, request done=6939/0, request doing=0/0) 302 [2024-06-06 11:28:17.981254] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=3/0, request done=6939/0, request doing=0/0) 303 [2024-06-06 11:28:17.982086] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 304 [2024-06-06 11:28:17.982348] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 305 [2024-06-06 11:28:17.982421] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=15] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 306 [2024-06-06 11:28:17.982688] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 307 [2024-06-06 11:28:18.063683] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 308 [2024-06-06 11:28:18.063733] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=34] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 309 [2024-06-06 11:28:18.091629] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502425-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 310 [2024-06-06 11:28:18.091684] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502425-0-0] [lt=51] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 311 [2024-06-06 11:28:18.110385] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602425-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 312 [2024-06-06 11:28:18.110431] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602425-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 313 [2024-06-06 11:28:18.112193] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702425-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 314 [2024-06-06 11:28:18.112222] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702425-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 315 [2024-06-06 11:28:18.163831] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 316 [2024-06-06 11:28:18.163859] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502425-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602425-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702425-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 317 [2024-06-06 11:28:18.163901] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=41][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 318 [2024-06-06 11:28:18.163908] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502425-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602425-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702425-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 319 [2024-06-06 11:28:18.264049] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=20] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 320 [2024-06-06 11:28:18.264171] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=82] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 321 [2024-06-06 11:28:18.289462] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502426-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 322 [2024-06-06 11:28:18.289548] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502426-0-0] [lt=81] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 323 [2024-06-06 11:28:18.310011] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602426-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 324 [2024-06-06 11:28:18.310060] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602426-0-0] [lt=45] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 325 [2024-06-06 11:28:18.311694] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702426-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 326 [2024-06-06 11:28:18.311726] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702426-0-0] [lt=30] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 327 [2024-06-06 11:28:18.364289] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=43][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 328 [2024-06-06 11:28:18.364327] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=37][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502426-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602426-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702426-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 329 [2024-06-06 11:28:18.364375] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=47][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 330 [2024-06-06 11:28:18.364381] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502426-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602426-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702426-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 331 [2024-06-06 11:28:18.464449] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 332 [2024-06-06 11:28:18.464499] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 333 [2024-06-06 11:28:18.487927] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502427-0-0] [lt=43] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 334 [2024-06-06 11:28:18.487963] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502427-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 335 [2024-06-06 11:28:18.516278] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602427-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 336 [2024-06-06 11:28:18.516310] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602427-0-0] [lt=30] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 337 [2024-06-06 11:28:18.525037] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702427-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 338 [2024-06-06 11:28:18.525066] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702427-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 339 [2024-06-06 11:28:18.564583] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 340 [2024-06-06 11:28:18.564615] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=31][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502427-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602427-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702427-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 341 [2024-06-06 11:28:18.564652] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 342 [2024-06-06 11:28:18.564664] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=10][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502427-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602427-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702427-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 343 [2024-06-06 11:28:18.664766] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=20] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 344 [2024-06-06 11:28:18.664817] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 345 [2024-06-06 11:28:18.699862] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502428-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 346 [2024-06-06 11:28:18.699905] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502428-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 347 [2024-06-06 11:28:18.716753] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 348 [2024-06-06 11:28:18.717004] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 349 [2024-06-06 11:28:18.717094] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 350 [2024-06-06 11:28:18.717455] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 351 [2024-06-06 11:28:18.724645] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602428-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 352 [2024-06-06 11:28:18.724692] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602428-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 353 [2024-06-06 11:28:18.726402] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702428-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 354 [2024-06-06 11:28:18.726427] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702428-0-0] [lt=23] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 355 [2024-06-06 11:28:18.764919] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 356 [2024-06-06 11:28:18.764946] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502428-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602428-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702428-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 357 [2024-06-06 11:28:18.764987] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=40][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 358 [2024-06-06 11:28:18.764993] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502428-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602428-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702428-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 359 [2024-06-06 11:28:18.819106] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=25] PNIO [ratelimit] time: 1717644498819096, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007612, add_bytes: 0 360 [2024-06-06 11:28:18.825847] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=30] PNIO [ratelimit] time: 1717644498825845, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007618, add_bytes: 0 361 [2024-06-06 11:28:18.865094] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 362 [2024-06-06 11:28:18.865134] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 363 [2024-06-06 11:28:18.917600] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502429-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 364 [2024-06-06 11:28:18.917663] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502429-0-0] [lt=56] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 365 [2024-06-06 11:28:18.937348] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=22] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 366 [2024-06-06 11:28:18.938270] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602429-0-0] [lt=23] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 367 [2024-06-06 11:28:18.938306] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602429-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 368 [2024-06-06 11:28:18.938841] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 369 [2024-06-06 11:28:18.938857] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=12] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 370 [2024-06-06 11:28:18.939388] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 371 [2024-06-06 11:28:18.940449] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702429-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 372 [2024-06-06 11:28:18.940474] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702429-0-0] [lt=23] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 373 [2024-06-06 11:28:18.965222] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=20][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 374 [2024-06-06 11:28:18.965244] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=22][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502429-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602429-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702429-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 375 [2024-06-06 11:28:18.965291] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=46][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 376 [2024-06-06 11:28:18.965296] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502429-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602429-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702429-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 377 [2024-06-06 11:28:18.980503] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=3/0, request done=6943/0, request doing=0/0) 378 [2024-06-06 11:28:18.980658] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=3/0, request done=6943/0, request doing=0/0) 379 [2024-06-06 11:28:18.981313] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=3/0, request done=6943/0, request doing=0/0) 380 [2024-06-06 11:28:18.981662] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=3/0, request done=6942/0, request doing=0/0) 381 [2024-06-06 11:28:18.982032] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=13] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 382 [2024-06-06 11:28:18.982111] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=13] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 383 [2024-06-06 11:28:18.982712] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=9] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 384 [2024-06-06 11:28:18.983348] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 385 [2024-06-06 11:28:19.065363] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 386 [2024-06-06 11:28:19.065428] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=42] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 387 [2024-06-06 11:28:19.106807] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050242A-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 388 [2024-06-06 11:28:19.106854] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050242A-0-0] [lt=44] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 389 [2024-06-06 11:28:19.132122] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060242A-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 390 [2024-06-06 11:28:19.132166] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060242A-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 391 [2024-06-06 11:28:19.134238] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070242A-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 392 [2024-06-06 11:28:19.134271] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070242A-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 393 [2024-06-06 11:28:19.165545] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=38][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 394 [2024-06-06 11:28:19.165577] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=31][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242A-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 395 [2024-06-06 11:28:19.165628] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=50][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 396 [2024-06-06 11:28:19.165637] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=7][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242A-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 397 [2024-06-06 11:28:19.265778] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=61] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 398 [2024-06-06 11:28:19.265837] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=35] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 399 [2024-06-06 11:28:19.649301] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050242B-0-0] [lt=21] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 400 [2024-06-06 11:28:19.649353] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050242B-0-0] [lt=48] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 401 [2024-06-06 11:28:19.672132] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060242B-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 402 [2024-06-06 11:28:19.672177] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060242B-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 403 [2024-06-06 11:28:19.674324] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070242B-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 404 [2024-06-06 11:28:19.674373] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070242B-0-0] [lt=47] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 405 [2024-06-06 11:28:19.716077] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 406 [2024-06-06 11:28:19.716351] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 407 [2024-06-06 11:28:19.716885] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 408 [2024-06-06 11:28:19.717043] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 409 [2024-06-06 11:28:19.766234] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 410 [2024-06-06 11:28:19.766266] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242B-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 411 [2024-06-06 11:28:19.766313] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=46][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 412 [2024-06-06 11:28:19.766321] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=7][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242B-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 413 [2024-06-06 11:28:19.826721] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=23] PNIO [ratelimit] time: 1717644499826717, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007621, add_bytes: 0 414 [2024-06-06 11:28:19.833457] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=22] PNIO [ratelimit] time: 1717644499833455, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007610, add_bytes: 0 415 [2024-06-06 11:28:19.866450] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=47] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 416 [2024-06-06 11:28:19.866511] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=37] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 417 [2024-06-06 11:28:19.894268] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050242C-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 418 [2024-06-06 11:28:19.894306] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050242C-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 419 [2024-06-06 11:28:19.911685] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060242C-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 420 [2024-06-06 11:28:19.911720] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060242C-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 421 [2024-06-06 11:28:19.913425] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070242C-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 422 [2024-06-06 11:28:19.913453] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070242C-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 423 [2024-06-06 11:28:19.938397] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=22] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 424 [2024-06-06 11:28:19.938875] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=13] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 425 [2024-06-06 11:28:19.938879] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 426 [2024-06-06 11:28:19.938950] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 427 [2024-06-06 11:28:19.966619] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 428 [2024-06-06 11:28:19.966644] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242C-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 429 [2024-06-06 11:28:19.966693] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=48][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 430 [2024-06-06 11:28:19.966702] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=8][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242C-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 431 [2024-06-06 11:28:19.980645] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=3/0, request done=6945/0, request doing=0/0) 432 [2024-06-06 11:28:19.980760] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=6945/0, request doing=0/0) 433 [2024-06-06 11:28:19.981291] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=15] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 434 [2024-06-06 11:28:19.981345] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=3/0, request done=6945/0, request doing=0/0) 435 [2024-06-06 11:28:19.981504] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=3/0, request done=6945/0, request doing=0/0) 436 [2024-06-06 11:28:19.981719] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 437 [2024-06-06 11:28:19.982296] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=11] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 438 [2024-06-06 11:28:19.983376] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 439 [2024-06-06 11:28:20.066814] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 440 [2024-06-06 11:28:20.066865] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=35] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 441 [2024-06-06 11:28:20.089439] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050242D-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 442 [2024-06-06 11:28:20.089478] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050242D-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 443 [2024-06-06 11:28:20.106239] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060242D-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 444 [2024-06-06 11:28:20.106277] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060242D-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 445 [2024-06-06 11:28:20.108404] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070242D-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 446 [2024-06-06 11:28:20.108430] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070242D-0-0] [lt=24] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 447 [2024-06-06 11:28:20.155044] INFO [TLOG] add_ls_and_fetch_until_the_progress_is_reached (ob_log_meta_data_fetcher.cpp:195) [50554][][T0][Y0-0000000000000000-0-0] [lt=5] ObLogFetcher check_progress is less than end_timestamp(tenant_id=1020, cur_progress=1717490923886536173, end_tstamp_ns=1717513200000000000) 448 [2024-06-06 11:28:20.155077] INFO [TLOG] print_stat_info (ob_log_task_pool.h:266) [50554][][T0][Y0-0000000000000000-0-0] [lt=27] [STAT] [TRANS_TASK_POOL] OUT=0/1000 FIXED=0/1000 DYNAMIC=0 PAGE_POOL=1000/1000 449 [2024-06-06 11:28:20.166989] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=47][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 450 [2024-06-06 11:28:20.167016] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242D-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 451 [2024-06-06 11:28:20.167052] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 452 [2024-06-06 11:28:20.167058] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242D-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 453 [2024-06-06 11:28:20.267154] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 454 [2024-06-06 11:28:20.267203] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 455 [2024-06-06 11:28:20.292065] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050242E-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 456 [2024-06-06 11:28:20.292111] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050242E-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 457 [2024-06-06 11:28:20.310855] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060242E-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 458 [2024-06-06 11:28:20.310904] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060242E-0-0] [lt=45] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 459 [2024-06-06 11:28:20.312924] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070242E-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 460 [2024-06-06 11:28:20.312954] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070242E-0-0] [lt=28] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 461 [2024-06-06 11:28:20.367310] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 462 [2024-06-06 11:28:20.367354] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=42][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242E-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 463 [2024-06-06 11:28:20.367391] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 464 [2024-06-06 11:28:20.367398] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242E-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 465 [2024-06-06 11:28:20.467498] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 466 [2024-06-06 11:28:20.467546] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 467 [2024-06-06 11:28:20.495052] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050242F-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 468 [2024-06-06 11:28:20.495099] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050242F-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 469 [2024-06-06 11:28:20.514045] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060242F-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 470 [2024-06-06 11:28:20.514093] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060242F-0-0] [lt=44] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 471 [2024-06-06 11:28:20.518413] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070242F-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 472 [2024-06-06 11:28:20.518442] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070242F-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 473 [2024-06-06 11:28:20.567649] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 474 [2024-06-06 11:28:20.567683] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242F-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 475 [2024-06-06 11:28:20.567723] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=39][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 476 [2024-06-06 11:28:20.567729] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=4][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050242F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060242F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070242F-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 477 [2024-06-06 11:28:20.667823] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=20] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 478 [2024-06-06 11:28:20.667872] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 479 [2024-06-06 11:28:20.692535] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502430-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 480 [2024-06-06 11:28:20.692572] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502430-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 481 [2024-06-06 11:28:20.712422] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602430-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 482 [2024-06-06 11:28:20.712451] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602430-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 483 [2024-06-06 11:28:20.714114] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702430-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 484 [2024-06-06 11:28:20.714138] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702430-0-0] [lt=23] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 485 [2024-06-06 11:28:20.716540] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 486 [2024-06-06 11:28:20.716918] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 487 [2024-06-06 11:28:20.717072] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 488 [2024-06-06 11:28:20.717116] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 489 [2024-06-06 11:28:20.767968] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 490 [2024-06-06 11:28:20.767996] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502430-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602430-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702430-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 491 [2024-06-06 11:28:20.768025] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 492 [2024-06-06 11:28:20.768030] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502430-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602430-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702430-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 493 [2024-06-06 11:28:20.834340] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=25] PNIO [ratelimit] time: 1717644500834337, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007620, add_bytes: 0 494 [2024-06-06 11:28:20.834370] INFO eloop_run (eloop.c:136) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=25] PNIO print mem cache info mem_freelists_str:0,0,0,0,0,0,0,0,0,, mod_report:DEFAULT:25,249,088 495 [2024-06-06 11:28:20.841073] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=17] PNIO [ratelimit] time: 1717644500841072, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007617, add_bytes: 0 496 [2024-06-06 11:28:20.868128] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 497 [2024-06-06 11:28:20.868189] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=37] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 498 [2024-06-06 11:28:20.896671] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502431-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 499 [2024-06-06 11:28:20.896717] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502431-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 500 [2024-06-06 11:28:20.920904] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602431-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 501 [2024-06-06 11:28:20.920943] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602431-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 502 [2024-06-06 11:28:20.922682] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702431-0-0] [lt=11] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 503 [2024-06-06 11:28:20.922712] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702431-0-0] [lt=29] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 504 [2024-06-06 11:28:20.937843] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=20] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 505 [2024-06-06 11:28:20.938369] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=11] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 506 [2024-06-06 11:28:20.938677] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 507 [2024-06-06 11:28:20.938896] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=9] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 508 [2024-06-06 11:28:20.968297] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 509 [2024-06-06 11:28:20.968325] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502431-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602431-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702431-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 510 [2024-06-06 11:28:20.968373] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=47][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 511 [2024-06-06 11:28:20.968379] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502431-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602431-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702431-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 512 [2024-06-06 11:28:20.980754] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=3/0, request done=6949/0, request doing=0/0) 513 [2024-06-06 11:28:20.980754] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=3/0, request done=6949/0, request doing=0/0) 514 [2024-06-06 11:28:20.980967] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=3/0, request done=6949/0, request doing=0/0) 515 [2024-06-06 11:28:20.981397] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=3/0, request done=6948/0, request doing=0/0) 516 [2024-06-06 11:28:20.981639] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 517 [2024-06-06 11:28:20.982076] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=13] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 518 [2024-06-06 11:28:20.982755] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 519 [2024-06-06 11:28:20.983351] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 520 [2024-06-06 11:28:21.010630] INFO [COMMON] handle (memory_dump.cpp:468) [50852][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=16] handle dump task(task={type_:2, dump_all_:false, p_context_:null, slot_idx_:0, dump_tenant_ctx_:false, tenant_id_:0, ctx_id_:0, p_chunk_:null}) 521 [2024-06-06 11:28:21.012640] INFO handle (memory_dump.cpp:589) [50852][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=26] statistics: 522 tenant_cnt: 3, max_chunk_cnt: 524288 523 tenant_id ctx_id chunk_cnt label_cnt segv_cnt 524 1 0 2 4 0 525 500 0 44 95 0 526 500 8 13 1 0 527 500 17 9 1 0 528 500 23 1 1 0 529 cost_time: 1899 530 [2024-06-06 11:28:21.068483] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 531 [2024-06-06 11:28:21.068540] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=34] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 532 [2024-06-06 11:28:21.088954] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502432-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 533 [2024-06-06 11:28:21.088992] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502432-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 534 [2024-06-06 11:28:21.107609] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602432-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 535 [2024-06-06 11:28:21.107638] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602432-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 536 [2024-06-06 11:28:21.109601] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702432-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 537 [2024-06-06 11:28:21.109626] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702432-0-0] [lt=23] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 538 [2024-06-06 11:28:21.168646] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 539 [2024-06-06 11:28:21.168676] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=29][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502432-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602432-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702432-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 540 [2024-06-06 11:28:21.168712] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 541 [2024-06-06 11:28:21.168718] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=4][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502432-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602432-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702432-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 542 [2024-06-06 11:28:21.268788] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 543 [2024-06-06 11:28:21.268836] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=29] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 544 [2024-06-06 11:28:21.290582] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502433-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 545 [2024-06-06 11:28:21.290627] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502433-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 546 [2024-06-06 11:28:21.314226] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602433-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 547 [2024-06-06 11:28:21.314280] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602433-0-0] [lt=50] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 548 [2024-06-06 11:28:21.315960] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702433-0-0] [lt=11] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 549 [2024-06-06 11:28:21.315989] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702433-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 550 [2024-06-06 11:28:21.368937] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 551 [2024-06-06 11:28:21.368963] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=25][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502433-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602433-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702433-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 552 [2024-06-06 11:28:21.369006] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=43][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 553 [2024-06-06 11:28:21.369012] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502433-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602433-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702433-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 554 [2024-06-06 11:28:21.469106] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 555 [2024-06-06 11:28:21.469155] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 556 [2024-06-06 11:28:21.493730] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502434-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 557 [2024-06-06 11:28:21.493784] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502434-0-0] [lt=48] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 558 [2024-06-06 11:28:21.510660] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602434-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 559 [2024-06-06 11:28:21.510704] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602434-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 560 [2024-06-06 11:28:21.512484] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702434-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 561 [2024-06-06 11:28:21.512513] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702434-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 562 [2024-06-06 11:28:21.569276] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 563 [2024-06-06 11:28:21.569345] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=68][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502434-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602434-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702434-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 564 [2024-06-06 11:28:21.569407] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=61][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 565 [2024-06-06 11:28:21.569415] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=7][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502434-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602434-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702434-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 566 [2024-06-06 11:28:21.669520] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 567 [2024-06-06 11:28:21.669578] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=39] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 568 [2024-06-06 11:28:21.693450] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502435-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 569 [2024-06-06 11:28:21.693495] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502435-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 570 [2024-06-06 11:28:21.712296] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602435-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 571 [2024-06-06 11:28:21.712353] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602435-0-0] [lt=53] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 572 [2024-06-06 11:28:21.714225] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702435-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 573 [2024-06-06 11:28:21.714246] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702435-0-0] [lt=20] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 574 [2024-06-06 11:28:21.716350] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 575 [2024-06-06 11:28:21.716353] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 576 [2024-06-06 11:28:21.717100] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 577 [2024-06-06 11:28:21.717145] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 578 [2024-06-06 11:28:21.769678] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 579 [2024-06-06 11:28:21.769712] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502435-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602435-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702435-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 580 [2024-06-06 11:28:21.769749] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 581 [2024-06-06 11:28:21.769755] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502435-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602435-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702435-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 582 [2024-06-06 11:28:21.841951] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=5] PNIO [ratelimit] time: 1717644501841948, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007611, add_bytes: 0 583 [2024-06-06 11:28:21.848687] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=27] PNIO [ratelimit] time: 1717644501848686, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007614, add_bytes: 0 584 [2024-06-06 11:28:21.869848] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 585 [2024-06-06 11:28:21.869895] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=29] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 586 [2024-06-06 11:28:21.902935] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=18] [STAT] [ALL_ZONE] INDEX=0/3 ZONE=az1 REGION=BJ STORAGE_TYPE=LOCAL VERSION=372 587 [2024-06-06 11:28:21.902962] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=20] [STAT] [ALL_ZONE] INDEX=1/3 ZONE=az2 REGION=BJ STORAGE_TYPE=LOCAL VERSION=372 588 [2024-06-06 11:28:21.902967] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_ZONE] INDEX=2/3 ZONE=az4 REGION=BJ STORAGE_TYPE=LOCAL VERSION=372 589 [2024-06-06 11:28:21.902972] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_ZONE] INDEX=0/3 ZONE=az1 ZONE_TYPE=ReadWrite VERSION=372 590 [2024-06-06 11:28:21.902976] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_ZONE] INDEX=1/3 ZONE=az2 ZONE_TYPE=ReadWrite VERSION=372 591 [2024-06-06 11:28:21.902980] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_ZONE] INDEX=2/3 ZONE=az4 ZONE_TYPE=ReadWrite VERSION=372 592 [2024-06-06 11:28:21.902983] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:454) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_ZONE] COUNT=3 VERSION=372 593 [2024-06-06 11:28:21.902991] INFO [TLOG] purge_stale_zone_records_ (ob_log_all_svr_cache.cpp:660) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_ZONE] [PURGE] PURGE_COUNT=0 CUR_COUNT=3 VERSION=372 594 [2024-06-06 11:28:21.907041] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_SERVER_LIST] INDEX=0/15 SERVER_ID=4 SERVER="10.50.54.157:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=372 595 [2024-06-06 11:28:21.907066] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=20] [STAT] [ALL_SERVER_LIST] INDEX=1/15 SERVER_ID=6 SERVER="10.50.54.245:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=372 596 [2024-06-06 11:28:21.907073] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [ALL_SERVER_LIST] INDEX=2/15 SERVER_ID=5 SERVER="10.50.56.27:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=372 597 [2024-06-06 11:28:21.907078] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=3/15 SERVER_ID=2 SERVER="10.50.56.28:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=372 598 [2024-06-06 11:28:21.907082] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=4/15 SERVER_ID=1 SERVER="10.50.73.94:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=372 599 [2024-06-06 11:28:21.907087] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_SERVER_LIST] INDEX=5/15 SERVER_ID=9 SERVER="10.52.18.156:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=372 600 [2024-06-06 11:28:21.907092] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=6/15 SERVER_ID=8 SERVER="10.52.18.158:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=372 601 [2024-06-06 11:28:21.907098] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_SERVER_LIST] INDEX=7/15 SERVER_ID=7 SERVER="10.52.18.180:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=372 602 [2024-06-06 11:28:21.907106] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=7] [STAT] [ALL_SERVER_LIST] INDEX=8/15 SERVER_ID=13 SERVER="10.53.0.225:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=372 603 [2024-06-06 11:28:21.907110] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=9/15 SERVER_ID=15 SERVER="10.53.73.35:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=372 604 [2024-06-06 11:28:21.907115] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=10/15 SERVER_ID=14 SERVER="10.53.73.37:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=372 605 [2024-06-06 11:28:21.907119] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=11/15 SERVER_ID=10 SERVER="10.88.10.155:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=372 606 [2024-06-06 11:28:21.907123] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=12/15 SERVER_ID=3 SERVER="10.88.10.156:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=372 607 [2024-06-06 11:28:21.907127] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=13/15 SERVER_ID=11 SERVER="10.88.20.97:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=372 608 [2024-06-06 11:28:21.907131] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=14/15 SERVER_ID=12 SERVER="10.88.8.206:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=372 609 [2024-06-06 11:28:21.907135] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:525) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] COUNT=15 VERSION=372 610 [2024-06-06 11:28:21.907144] INFO [TLOG] purge_stale_records_ (ob_log_all_svr_cache.cpp:646) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=2] [STAT] [ALL_SERVER_LIST] [PURGE] PURGE_COUNT=0 CUR_COUNT=15 VERSION=372 611 [2024-06-06 11:28:21.917299] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502436-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 612 [2024-06-06 11:28:21.917344] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502436-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 613 [2024-06-06 11:28:21.935749] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602436-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 614 [2024-06-06 11:28:21.935795] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602436-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 615 [2024-06-06 11:28:21.937879] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=20] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 616 [2024-06-06 11:28:21.938401] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 617 [2024-06-06 11:28:21.938409] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 618 [2024-06-06 11:28:21.938660] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702436-0-0] [lt=11] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 619 [2024-06-06 11:28:21.938679] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702436-0-0] [lt=17] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 620 [2024-06-06 11:28:21.939706] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 621 [2024-06-06 11:28:21.969992] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 622 [2024-06-06 11:28:21.970016] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502436-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602436-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702436-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 623 [2024-06-06 11:28:21.970053] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 624 [2024-06-06 11:28:21.970059] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502436-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602436-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702436-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 625 [2024-06-06 11:28:21.971311] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24432683789}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}]) 626 [2024-06-06 11:28:21.971343] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=31] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24432683789}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24432683911}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}]) 627 [2024-06-06 11:28:21.971353] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=9] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24432683789}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24432683911}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}, {server:"10.88.8.206:2882", ranges:[LSN:{24291926016, 24432683789}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}]) 628 [2024-06-06 11:28:21.971365] INFO [TLOG] get_ls_svr_list_ (ob_log_route_service.cpp:809) [50969][LogRouter][T0][Y0-0000000000000000-0-0] [lt=7] get_ls_svr_list_ done(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}, svr_list={next_svr_index:0, svr_num:3, svr_items:[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24432683789}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.88.8.206:2882", ranges:[LSN:{24291926016, 24432683789}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24432683911}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}]}, tg_id_=259) 629 [2024-06-06 11:28:21.980720] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=3/0, request done=6953/0, request doing=0/0) 630 [2024-06-06 11:28:21.980800] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=3/0, request done=6953/0, request doing=0/0) 631 [2024-06-06 11:28:21.981350] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=22] [RPC EASY STAT](log_str=conn count=3/0, request done=6952/0, request doing=0/0) 632 [2024-06-06 11:28:21.981361] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=19] [RPC EASY STAT](log_str=conn count=3/0, request done=6952/0, request doing=0/0) 633 [2024-06-06 11:28:21.981846] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 634 [2024-06-06 11:28:21.982085] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=15] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 635 [2024-06-06 11:28:21.982783] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=13] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 636 [2024-06-06 11:28:21.983387] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 637 [2024-06-06 11:28:22.070165] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=20] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 638 [2024-06-06 11:28:22.070254] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=50] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 639 [2024-06-06 11:28:22.131208] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502437-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 640 [2024-06-06 11:28:22.131257] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502437-0-0] [lt=44] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 641 [2024-06-06 11:28:22.154475] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602437-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 642 [2024-06-06 11:28:22.154521] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602437-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 643 [2024-06-06 11:28:22.160270] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702437-0-0] [lt=10] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 644 [2024-06-06 11:28:22.160303] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702437-0-0] [lt=30] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 645 [2024-06-06 11:28:22.170359] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 646 [2024-06-06 11:28:22.170383] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502437-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602437-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702437-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 647 [2024-06-06 11:28:22.170424] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=41][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 648 [2024-06-06 11:28:22.170436] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=10][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502437-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602437-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702437-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 649 [2024-06-06 11:28:22.270531] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 650 [2024-06-06 11:28:22.270579] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 651 [2024-06-06 11:28:22.343858] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502438-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 652 [2024-06-06 11:28:22.343906] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502438-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 653 [2024-06-06 11:28:22.366599] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602438-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 654 [2024-06-06 11:28:22.366640] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602438-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 655 [2024-06-06 11:28:22.368508] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702438-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 656 [2024-06-06 11:28:22.368545] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702438-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 657 [2024-06-06 11:28:22.370668] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 658 [2024-06-06 11:28:22.370690] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502438-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602438-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702438-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 659 [2024-06-06 11:28:22.370721] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 660 [2024-06-06 11:28:22.370727] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502438-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602438-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702438-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 661 [2024-06-06 11:28:22.470829] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 662 [2024-06-06 11:28:22.470878] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=29] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 663 [2024-06-06 11:28:22.534656] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502439-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 664 [2024-06-06 11:28:22.534711] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502439-0-0] [lt=51] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 665 [2024-06-06 11:28:22.555990] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602439-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 666 [2024-06-06 11:28:22.556055] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602439-0-0] [lt=61] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 667 [2024-06-06 11:28:22.557970] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702439-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 668 [2024-06-06 11:28:22.557999] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702439-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 669 [2024-06-06 11:28:22.570959] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 670 [2024-06-06 11:28:22.570986] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502439-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602439-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702439-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 671 [2024-06-06 11:28:22.571019] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 672 [2024-06-06 11:28:22.571025] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502439-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602439-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702439-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 673 [2024-06-06 11:28:22.671132] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 674 [2024-06-06 11:28:22.671180] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 675 [2024-06-06 11:28:22.716350] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 676 [2024-06-06 11:28:22.716404] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 677 [2024-06-06 11:28:22.716590] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 678 [2024-06-06 11:28:22.716793] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 679 [2024-06-06 11:28:22.744380] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050243A-0-0] [lt=24] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 680 [2024-06-06 11:28:22.744419] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050243A-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 681 [2024-06-06 11:28:22.766023] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060243A-0-0] [lt=23] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 682 [2024-06-06 11:28:22.766052] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060243A-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 683 [2024-06-06 11:28:22.768296] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070243A-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 684 [2024-06-06 11:28:22.768324] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070243A-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 685 [2024-06-06 11:28:22.771271] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 686 [2024-06-06 11:28:22.771291] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=20][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243A-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 687 [2024-06-06 11:28:22.771319] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=27][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 688 [2024-06-06 11:28:22.771325] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243A-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 689 [2024-06-06 11:28:22.849571] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=23] PNIO [ratelimit] time: 1717644502849567, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007619, add_bytes: 0 690 [2024-06-06 11:28:22.856303] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=20] PNIO [ratelimit] time: 1717644502856301, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007615, add_bytes: 0 691 [2024-06-06 11:28:22.871430] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=34] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 692 [2024-06-06 11:28:22.871474] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=25] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 693 [2024-06-06 11:28:22.937425] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 694 [2024-06-06 11:28:22.938179] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 695 [2024-06-06 11:28:22.938348] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=11] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 696 [2024-06-06 11:28:22.938428] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 697 [2024-06-06 11:28:22.942397] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050243B-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 698 [2024-06-06 11:28:22.942439] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050243B-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 699 [2024-06-06 11:28:22.965398] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060243B-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 700 [2024-06-06 11:28:22.965438] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060243B-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 701 [2024-06-06 11:28:22.968495] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070243B-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 702 [2024-06-06 11:28:22.968523] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070243B-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 703 [2024-06-06 11:28:22.971566] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 704 [2024-06-06 11:28:22.971584] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=18][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243B-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 705 [2024-06-06 11:28:22.971618] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 706 [2024-06-06 11:28:22.971626] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243B-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 707 [2024-06-06 11:28:22.980565] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=3/0, request done=6957/0, request doing=0/0) 708 [2024-06-06 11:28:22.981162] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 709 [2024-06-06 11:28:22.981446] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=6956/0, request doing=0/0) 710 [2024-06-06 11:28:22.981446] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=3/0, request done=6956/0, request doing=0/0) 711 [2024-06-06 11:28:22.981448] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=3/0, request done=6956/0, request doing=0/0) 712 [2024-06-06 11:28:22.982214] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 713 [2024-06-06 11:28:22.982399] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=11] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 714 [2024-06-06 11:28:22.983350] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 715 [2024-06-06 11:28:23.071728] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 716 [2024-06-06 11:28:23.071799] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=46] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 717 [2024-06-06 11:28:23.151648] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050243C-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 718 [2024-06-06 11:28:23.151694] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050243C-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 719 [2024-06-06 11:28:23.176822] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060243C-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 720 [2024-06-06 11:28:23.176860] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060243C-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 721 [2024-06-06 11:28:23.178769] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070243C-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 722 [2024-06-06 11:28:23.178797] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070243C-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 723 [2024-06-06 11:28:23.271971] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 724 [2024-06-06 11:28:23.271999] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243C-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 725 [2024-06-06 11:28:23.272034] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 726 [2024-06-06 11:28:23.272040] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243C-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 727 [2024-06-06 11:28:23.372131] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 728 [2024-06-06 11:28:23.372180] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 729 [2024-06-06 11:28:23.401289] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050243D-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 730 [2024-06-06 11:28:23.401325] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050243D-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 731 [2024-06-06 11:28:23.426022] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060243D-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 732 [2024-06-06 11:28:23.426074] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060243D-0-0] [lt=47] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 733 [2024-06-06 11:28:23.427668] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070243D-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 734 [2024-06-06 11:28:23.427696] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070243D-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 735 [2024-06-06 11:28:23.472281] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 736 [2024-06-06 11:28:23.472310] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243D-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 737 [2024-06-06 11:28:23.472353] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=42][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 738 [2024-06-06 11:28:23.472360] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243D-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 739 [2024-06-06 11:28:23.572460] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 740 [2024-06-06 11:28:23.572517] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=39] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 741 [2024-06-06 11:28:23.595277] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050243E-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 742 [2024-06-06 11:28:23.595322] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050243E-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 743 [2024-06-06 11:28:23.612343] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060243E-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 744 [2024-06-06 11:28:23.612377] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060243E-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 745 [2024-06-06 11:28:23.614071] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070243E-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 746 [2024-06-06 11:28:23.614097] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070243E-0-0] [lt=25] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 747 [2024-06-06 11:28:23.672619] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=29][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 748 [2024-06-06 11:28:23.672655] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=34][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243E-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 749 [2024-06-06 11:28:23.672700] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=44][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 750 [2024-06-06 11:28:23.672706] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243E-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 751 [2024-06-06 11:28:23.716565] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 752 [2024-06-06 11:28:23.717390] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 753 [2024-06-06 11:28:23.717436] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 754 [2024-06-06 11:28:23.717616] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 755 [2024-06-06 11:28:23.772795] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 756 [2024-06-06 11:28:23.772837] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 757 [2024-06-06 11:28:23.795283] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050243F-0-0] [lt=23] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 758 [2024-06-06 11:28:23.795321] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050243F-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 759 [2024-06-06 11:28:23.815811] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060243F-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 760 [2024-06-06 11:28:23.815854] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060243F-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 761 [2024-06-06 11:28:23.817615] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070243F-0-0] [lt=11] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 762 [2024-06-06 11:28:23.817646] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070243F-0-0] [lt=29] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 763 [2024-06-06 11:28:23.857187] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=35] PNIO [ratelimit] time: 1717644503857184, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007617, add_bytes: 0 764 [2024-06-06 11:28:23.863919] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=18] PNIO [ratelimit] time: 1717644503863918, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007617, add_bytes: 0 765 [2024-06-06 11:28:23.872935] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 766 [2024-06-06 11:28:23.872957] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243F-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 767 [2024-06-06 11:28:23.872994] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 768 [2024-06-06 11:28:23.873000] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050243F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060243F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070243F-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 769 [2024-06-06 11:28:23.938133] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=17] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 770 [2024-06-06 11:28:23.938266] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=17] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 771 [2024-06-06 11:28:23.938451] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 772 [2024-06-06 11:28:23.939112] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 773 [2024-06-06 11:28:23.973092] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 774 [2024-06-06 11:28:23.973138] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 775 [2024-06-06 11:28:23.980788] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=6960/0, request doing=0/0) 776 [2024-06-06 11:28:23.980976] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=3/0, request done=6959/0, request doing=0/0) 777 [2024-06-06 11:28:23.981261] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=19] [RPC EASY STAT](log_str=conn count=3/0, request done=6959/0, request doing=1/0) 778 [2024-06-06 11:28:23.981473] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=3/0, request done=6959/0, request doing=0/0) 779 [2024-06-06 11:28:23.981795] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 780 [2024-06-06 11:28:23.981913] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 781 [2024-06-06 11:28:23.982236] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 782 [2024-06-06 11:28:23.983271] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 783 [2024-06-06 11:28:23.997794] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502440-0-0] [lt=23] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 784 [2024-06-06 11:28:23.997830] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502440-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 785 [2024-06-06 11:28:24.018529] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602440-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 786 [2024-06-06 11:28:24.018572] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602440-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 787 [2024-06-06 11:28:24.020168] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702440-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 788 [2024-06-06 11:28:24.020196] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702440-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 789 [2024-06-06 11:28:24.073248] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 790 [2024-06-06 11:28:24.073276] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502440-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602440-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702440-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 791 [2024-06-06 11:28:24.073318] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=41][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 792 [2024-06-06 11:28:24.073325] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502440-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602440-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702440-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 793 [2024-06-06 11:28:24.173426] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 794 [2024-06-06 11:28:24.173477] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 795 [2024-06-06 11:28:24.202299] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502441-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 796 [2024-06-06 11:28:24.202344] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502441-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 797 [2024-06-06 11:28:24.226741] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602441-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 798 [2024-06-06 11:28:24.226770] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602441-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 799 [2024-06-06 11:28:24.228621] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702441-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 800 [2024-06-06 11:28:24.228647] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702441-0-0] [lt=24] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 801 [2024-06-06 11:28:24.273567] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 802 [2024-06-06 11:28:24.273590] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=22][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502441-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602441-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702441-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 803 [2024-06-06 11:28:24.273627] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=37][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 804 [2024-06-06 11:28:24.273633] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502441-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602441-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702441-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 805 [2024-06-06 11:28:24.373725] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 806 [2024-06-06 11:28:24.373786] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=41] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 807 [2024-06-06 11:28:24.396380] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502442-0-0] [lt=26] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 808 [2024-06-06 11:28:24.396416] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502442-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 809 [2024-06-06 11:28:24.416979] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602442-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 810 [2024-06-06 11:28:24.417019] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602442-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 811 [2024-06-06 11:28:24.418574] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702442-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 812 [2024-06-06 11:28:24.418604] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702442-0-0] [lt=28] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 813 [2024-06-06 11:28:24.473882] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=22][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 814 [2024-06-06 11:28:24.473913] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502442-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602442-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702442-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 815 [2024-06-06 11:28:24.473950] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=37][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 816 [2024-06-06 11:28:24.473966] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=15][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502442-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602442-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702442-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 817 [2024-06-06 11:28:24.574064] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 818 [2024-06-06 11:28:24.574115] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 819 [2024-06-06 11:28:24.601078] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502443-0-0] [lt=19] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 820 [2024-06-06 11:28:24.601125] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502443-0-0] [lt=44] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 821 [2024-06-06 11:28:24.621801] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602443-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 822 [2024-06-06 11:28:24.621847] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602443-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 823 [2024-06-06 11:28:24.631564] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702443-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 824 [2024-06-06 11:28:24.631595] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702443-0-0] [lt=29] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 825 [2024-06-06 11:28:24.674212] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 826 [2024-06-06 11:28:24.674237] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502443-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602443-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702443-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 827 [2024-06-06 11:28:24.674273] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 828 [2024-06-06 11:28:24.674279] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502443-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602443-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702443-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 829 [2024-06-06 11:28:24.716438] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 830 [2024-06-06 11:28:24.716951] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 831 [2024-06-06 11:28:24.717003] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 832 [2024-06-06 11:28:24.717346] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 833 [2024-06-06 11:28:24.774383] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 834 [2024-06-06 11:28:24.774435] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 835 [2024-06-06 11:28:24.794661] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502444-0-0] [lt=19] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 836 [2024-06-06 11:28:24.794703] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502444-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 837 [2024-06-06 11:28:24.820475] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602444-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 838 [2024-06-06 11:28:24.820514] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602444-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 839 [2024-06-06 11:28:24.821996] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702444-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 840 [2024-06-06 11:28:24.822012] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702444-0-0] [lt=16] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 841 [2024-06-06 11:28:24.864797] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=25] PNIO [ratelimit] time: 1717644504864794, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007610, add_bytes: 0 842 [2024-06-06 11:28:24.871538] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=16] PNIO [ratelimit] time: 1717644504871536, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007618, add_bytes: 0 843 [2024-06-06 11:28:24.874532] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 844 [2024-06-06 11:28:24.874555] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=22][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502444-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602444-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702444-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 845 [2024-06-06 11:28:24.874595] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=39][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 846 [2024-06-06 11:28:24.874601] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502444-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602444-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702444-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 847 [2024-06-06 11:28:24.937577] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 848 [2024-06-06 11:28:24.938253] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 849 [2024-06-06 11:28:24.938418] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 850 [2024-06-06 11:28:24.939007] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=24] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 851 [2024-06-06 11:28:24.974698] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 852 [2024-06-06 11:28:24.974740] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 853 [2024-06-06 11:28:24.980641] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=3/0, request done=6963/0, request doing=0/0) 854 [2024-06-06 11:28:24.980849] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=3/0, request done=6963/0, request doing=0/0) 855 [2024-06-06 11:28:24.980874] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=3/0, request done=6963/0, request doing=1/0) 856 [2024-06-06 11:28:24.981061] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=3/0, request done=6963/0, request doing=0/0) 857 [2024-06-06 11:28:24.981818] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 858 [2024-06-06 11:28:24.982263] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 859 [2024-06-06 11:28:24.982941] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 860 [2024-06-06 11:28:24.983303] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 861 [2024-06-06 11:28:24.997273] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502445-0-0] [lt=21] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 862 [2024-06-06 11:28:24.997302] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502445-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 863 [2024-06-06 11:28:25.016601] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602445-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 864 [2024-06-06 11:28:25.016636] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602445-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 865 [2024-06-06 11:28:25.018357] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702445-0-0] [lt=9] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 866 [2024-06-06 11:28:25.018382] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702445-0-0] [lt=24] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 867 [2024-06-06 11:28:25.074810] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 868 [2024-06-06 11:28:25.074860] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=49][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502445-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602445-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702445-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 869 [2024-06-06 11:28:25.074902] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=41][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 870 [2024-06-06 11:28:25.074910] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502445-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602445-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702445-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 871 [2024-06-06 11:28:25.175034] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=41] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 872 [2024-06-06 11:28:25.175119] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=62] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 873 [2024-06-06 11:28:25.201946] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502446-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 874 [2024-06-06 11:28:25.201999] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502446-0-0] [lt=49] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 875 [2024-06-06 11:28:25.222487] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602446-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 876 [2024-06-06 11:28:25.222544] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602446-0-0] [lt=53] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 877 [2024-06-06 11:28:25.225599] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702446-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 878 [2024-06-06 11:28:25.225640] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702446-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 879 [2024-06-06 11:28:25.275211] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 880 [2024-06-06 11:28:25.275236] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502446-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602446-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702446-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 881 [2024-06-06 11:28:25.275270] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 882 [2024-06-06 11:28:25.275277] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502446-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602446-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702446-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 883 [2024-06-06 11:28:25.375350] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 884 [2024-06-06 11:28:25.375405] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=37] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 885 [2024-06-06 11:28:25.395851] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502447-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 886 [2024-06-06 11:28:25.395887] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502447-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 887 [2024-06-06 11:28:25.416670] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602447-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 888 [2024-06-06 11:28:25.416704] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602447-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 889 [2024-06-06 11:28:25.418353] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702447-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 890 [2024-06-06 11:28:25.418381] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702447-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 891 [2024-06-06 11:28:25.475501] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 892 [2024-06-06 11:28:25.475530] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502447-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602447-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702447-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 893 [2024-06-06 11:28:25.475567] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 894 [2024-06-06 11:28:25.475573] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502447-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602447-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702447-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 895 [2024-06-06 11:28:25.575676] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 896 [2024-06-06 11:28:25.575737] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=38] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 897 [2024-06-06 11:28:25.600617] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502448-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 898 [2024-06-06 11:28:25.600663] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502448-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 899 [2024-06-06 11:28:25.619444] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602448-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 900 [2024-06-06 11:28:25.619485] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602448-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 901 [2024-06-06 11:28:25.621197] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702448-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 902 [2024-06-06 11:28:25.621227] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702448-0-0] [lt=28] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 903 [2024-06-06 11:28:25.675837] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=38][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 904 [2024-06-06 11:28:25.675865] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502448-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602448-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702448-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 905 [2024-06-06 11:28:25.675910] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=44][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 906 [2024-06-06 11:28:25.675917] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502448-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602448-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702448-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 907 [2024-06-06 11:28:25.716357] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 908 [2024-06-06 11:28:25.716992] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 909 [2024-06-06 11:28:25.717237] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 910 [2024-06-06 11:28:25.717377] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 911 [2024-06-06 11:28:25.776008] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 912 [2024-06-06 11:28:25.776048] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=24] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 913 [2024-06-06 11:28:25.801004] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-0000000000502449-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 914 [2024-06-06 11:28:25.801038] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-0000000000502449-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 915 [2024-06-06 11:28:25.821794] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-0000000000602449-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 916 [2024-06-06 11:28:25.821827] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-0000000000602449-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 917 [2024-06-06 11:28:25.823333] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-0000000000702449-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 918 [2024-06-06 11:28:25.823364] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-0000000000702449-0-0] [lt=30] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 919 [2024-06-06 11:28:25.872413] INFO eloop_run (eloop.c:127) [50944][pnio1][T0][Y0-0000000000000000-0-0] [lt=23] PNIO [ratelimit] time: 1717644505872411, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007617, add_bytes: 0 920 [2024-06-06 11:28:25.876127] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 921 [2024-06-06 11:28:25.876151] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502449-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602449-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702449-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 922 [2024-06-06 11:28:25.876186] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 923 [2024-06-06 11:28:25.876193] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000502449-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000602449-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-0000000000702449-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 924 [2024-06-06 11:28:25.879152] INFO eloop_run (eloop.c:127) [50948][pnio2][T0][Y0-0000000000000000-0-0] [lt=20] PNIO [ratelimit] time: 1717644505879151, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007615, add_bytes: 0 925 [2024-06-06 11:28:25.938048] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50963][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=24] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 926 [2024-06-06 11:28:25.938061] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50965][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=12] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 927 [2024-06-06 11:28:25.938983] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50967][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 928 [2024-06-06 11:28:25.939036] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50961][BatchIO][T0][Y0-0000000000000000-0-0] [lt=11] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 929 [2024-06-06 11:28:25.976290] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 930 [2024-06-06 11:28:25.976343] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=30] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 931 [2024-06-06 11:28:25.980960] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50972][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=3/0, request done=6967/0, request doing=0/0) 932 [2024-06-06 11:28:25.981185] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50971][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=3/0, request done=6967/0, request doing=0/0) 933 [2024-06-06 11:28:25.981470] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50974][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=3/0, request done=6966/0, request doing=1/0) 934 [2024-06-06 11:28:25.981493] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50973][RpcIO][T0][Y0-0000000000000000-0-0] [lt=19] [RPC EASY STAT](log_str=conn count=3/0, request done=6967/0, request doing=0/0) 935 [2024-06-06 11:28:25.981844] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50978][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 936 [2024-06-06 11:28:25.982288] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [50980][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=13] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 937 [2024-06-06 11:28:25.982472] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [50976][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 938 [2024-06-06 11:28:25.983337] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50982][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 939 [2024-06-06 11:28:26.005616] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050244A-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 940 [2024-06-06 11:28:26.005655] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050244A-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 941 [2024-06-06 11:28:26.030335] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060244A-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 942 [2024-06-06 11:28:26.030378] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060244A-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 943 [2024-06-06 11:28:26.032182] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070244A-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 944 [2024-06-06 11:28:26.032210] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070244A-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 945 [2024-06-06 11:28:26.076443] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 946 [2024-06-06 11:28:26.076470] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050244A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060244A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070244A-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 947 [2024-06-06 11:28:26.076512] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=42][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 948 [2024-06-06 11:28:26.076519] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050244A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060244A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070244A-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 949 [2024-06-06 11:28:26.145839] INFO [TLOG.FETCHER] print_stat (ob_log_fetch_stream_container_mgr.cpp:181) [51002][][T0][Y0-0000000000000000-0-0] [lt=22] [STAT] [FS_CONTAINER_POOL] USED=1 FREE=0 FIXED=16 DYNAMIC=0 950 [2024-06-06 11:28:26.145863] INFO [TLOG.FETCHER] print_stat (ob_log_fetch_stream_pool.cpp:76) [51002][][T0][Y0-0000000000000000-0-0] [lt=22] [STAT] [FETCH_STREAM_POOL] USED=0 FREE=0 FIXED=16 DYNAMIC=0 951 [2024-06-06 11:28:26.145869] INFO [TLOG.FETCHER] print_stat (ob_log_fetch_log_rpc.cpp:1400) [51002][][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [RPC_RESULT_POOL] USED=0 FREE=0 FIXED=16 DYNAMIC=0 952 [2024-06-06 11:28:26.145909] INFO [TLOG.FETCHER] operator() (ob_log_fetch_stream_container_mgr.cpp:230) [51002][][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [FETCH_STREAM] TENANT=1020, TRAFFIC=0.00B/sec 953 [2024-06-06 11:28:26.145928] INFO [TLOG.FETCHER] update_execution_time_ (ob_log_part_progress_controller.cpp:305) [51002][][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [GET_MIN_PROGRESS] AVG_TIME=1 LS_COUNT=1 SCAN_COUNT=6 954 [2024-06-06 11:28:26.145943] INFO [TLOG.FETCHER] print_fetcher_stat_ (ob_log_fetcher.cpp:840) [51002][][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [FETCHER](upper_limit="[2024-06-04 16:49:13.886536]", dml_progress_limit_sec=30, fetcher_delay="[42:39:42.259396]") 955 [2024-06-06 11:28:26.146042] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:429) [51002][][T0][Y0-0000000000000000-0-0] [lt=8] [STAT] slow fetch progress start. ls_count=1/1 956 [2024-06-06 11:28:26.146056] INFO [TLOG.FETCHER] print_fetch_progress (ob_log_ls_fetch_ctx.cpp:1454) [51002][][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] slow fetch progress idx=0/1 tls_id={tenant_id:1020, ls_id:{id:1}} mod=IDLE_POOL discarded=0 delay=[42:39:42.259505] tps=0.00 progress={next_lsn:{lsn:18446744073709551615}, log_progress:"[2024-06-04 16:48:43.886536]", log_touch_tstamp:"[2024-06-06 10:57:10.051911]"} 957 [2024-06-06 11:28:26.146066] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:436) [51002][][T0][Y0-0000000000000000-0-0] [lt=9] [STAT] slow fetch progress end 958 [2024-06-06 11:28:26.146070] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:441) [51002][][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] slow dispatch progress start. ls_count=1/1 959 [2024-06-06 11:28:26.146077] INFO [TLOG.FETCHER] print_dispatch_progress (ob_log_ls_fetch_ctx.cpp:1476) [51002][][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] slow dispatch progress idx=0/1 tls_id={tenant_id:1020, ls_id:{id:1}} delay=[42:39:42.259505] pending_task(queue/total)=0/0 dispatch_progress=[2024-06-04 16:48:43.886536] last_dispatch_log_lsn=18446744073709551615 next_task=INVALID next_trans(log_lsn=18446744073709551615,committed=0,ready_to_commit=0,global_version=[1970-01-01 08:00:00.000000]) checkpoint=[1970-01-01 08:00:00.000000] 960 [2024-06-06 11:28:26.146083] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:448) [51002][][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] slow dispatch progress end 961 [2024-06-06 11:28:26.176615] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 962 [2024-06-06 11:28:26.176660] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=27] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 963 [2024-06-06 11:28:26.197072] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050244B-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 964 [2024-06-06 11:28:26.197108] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050244B-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 965 [2024-06-06 11:28:26.215975] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060244B-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 966 [2024-06-06 11:28:26.216017] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060244B-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 967 [2024-06-06 11:28:26.218088] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070244B-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 968 [2024-06-06 11:28:26.218115] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070244B-0-0] [lt=25] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 969 [2024-06-06 11:28:26.276760] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 970 [2024-06-06 11:28:26.276789] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050244B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060244B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070244B-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 971 [2024-06-06 11:28:26.276825] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 972 [2024-06-06 11:28:26.276831] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050244B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060244B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070244B-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 973 [2024-06-06 11:28:26.376932] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 974 [2024-06-06 11:28:26.376995] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=44] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 975 [2024-06-06 11:28:26.397066] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050244C-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 976 [2024-06-06 11:28:26.397124] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050244C-0-0] [lt=54] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 977 [2024-06-06 11:28:26.416516] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060244C-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 978 [2024-06-06 11:28:26.416550] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060244C-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 979 [2024-06-06 11:28:26.418191] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070244C-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 980 [2024-06-06 11:28:26.418219] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070244C-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 981 [2024-06-06 11:28:26.477089] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=23][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 982 [2024-06-06 11:28:26.477116] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050244C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060244C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070244C-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 983 [2024-06-06 11:28:26.477151] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 984 [2024-06-06 11:28:26.477157] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050244C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060244C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070244C-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 985 [2024-06-06 11:28:26.577256] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 986 [2024-06-06 11:28:26.577303] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-04 16:48:43.886536]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 987 [2024-06-06 11:28:26.599775] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50987][][T0][YC4740A5818EC-000000000050244D-0-0] [lt=21] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 988 [2024-06-06 11:28:26.599807] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50987][][T0][YC4740A5818EC-000000000050244D-0-0] [lt=29] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 989 [2024-06-06 11:28:26.621474] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50985][][T0][YC4740A5818EC-000000000060244D-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 990 [2024-06-06 11:28:26.621501] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50985][][T0][YC4740A5818EC-000000000060244D-0-0] [lt=25] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 991 [2024-06-06 11:28:26.623024] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [50984][][T0][YC4740A5818EC-000000000070244D-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717490923886536174}], client_id:{client_addr:"10.88.24.236:50292", client_pid:50292}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 992 [2024-06-06 11:28:26.623051] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [50984][][T0][YC4740A5818EC-000000000070244D-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717490923886536174, delta=-1717490923886536175, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 993 [2024-06-06 11:28:26.677413] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=32][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 994 [2024-06-06 11:28:26.677447] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=33][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717490923886536174, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050244D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060244D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070244D-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 995 [2024-06-06 11:28:26.677485] ERROR issue_dba_error (ob_log.cpp:1875) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=37][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 996 [2024-06-06 11:28:26.677491] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [50988][][T0][YC4740A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717490923886536174, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000050244D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000060244D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:YC4740A5818EC-000000000070244D-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7f7de0cc3ea5 0x7f7de02ddb0d 997 [2024-06-06 11:28:26.716361] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50956][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 998 [2024-06-06 11:28:26.717031] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50957][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 999 [2024-06-06 11:28:26.717129] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50958][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 1000 [2024-06-06 11:28:26.717346] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [50959][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0)