1 [2024-06-06 10:29:35.640857] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 2 [2024-06-06 10:29:35.640884] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501214-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601214-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701214-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 3 [2024-06-06 10:29:35.640922] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=37][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 4 [2024-06-06 10:29:35.640928] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501214-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601214-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701214-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 5 [2024-06-06 10:29:35.729800] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 6 [2024-06-06 10:29:35.730037] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=19] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 7 [2024-06-06 10:29:35.730354] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 8 [2024-06-06 10:29:35.730392] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 9 [2024-06-06 10:29:35.741032] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 10 [2024-06-06 10:29:35.741076] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 11 [2024-06-06 10:29:35.760558] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=19] PNIO [ratelimit] time: 1717640975760556, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007615, add_bytes: 0 12 [2024-06-06 10:29:35.767982] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501215-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 13 [2024-06-06 10:29:35.768020] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501215-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 14 [2024-06-06 10:29:35.777455] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=20] PNIO [ratelimit] time: 1717640975777452, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007615, add_bytes: 0 15 [2024-06-06 10:29:35.793256] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601215-0-0] [lt=19] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 16 [2024-06-06 10:29:35.793299] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601215-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 17 [2024-06-06 10:29:35.795342] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701215-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 18 [2024-06-06 10:29:35.795377] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701215-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 19 [2024-06-06 10:29:35.841175] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 20 [2024-06-06 10:29:35.841205] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=29][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501215-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601215-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701215-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 21 [2024-06-06 10:29:35.841243] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=37][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 22 [2024-06-06 10:29:35.841254] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=10][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501215-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601215-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701215-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 23 [2024-06-06 10:29:35.941354] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 24 [2024-06-06 10:29:35.941409] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 25 [2024-06-06 10:29:35.961490] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501216-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 26 [2024-06-06 10:29:35.961539] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501216-0-0] [lt=45] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 27 [2024-06-06 10:29:35.968677] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=23] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 28 [2024-06-06 10:29:35.969546] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 29 [2024-06-06 10:29:35.970352] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 30 [2024-06-06 10:29:35.970492] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 31 [2024-06-06 10:29:35.987034] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601216-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 32 [2024-06-06 10:29:35.987076] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601216-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 33 [2024-06-06 10:29:35.992203] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701216-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 34 [2024-06-06 10:29:35.992237] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701216-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 35 [2024-06-06 10:29:36.008258] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=3/0, request done=3473/0, request doing=0/0) 36 [2024-06-06 10:29:36.008524] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=3/0, request done=3472/0, request doing=0/0) 37 [2024-06-06 10:29:36.008538] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=3/0, request done=3472/0, request doing=0/0) 38 [2024-06-06 10:29:36.009079] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=3/0, request done=3473/0, request doing=0/0) 39 [2024-06-06 10:29:36.009836] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 40 [2024-06-06 10:29:36.010205] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 41 [2024-06-06 10:29:36.010536] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 42 [2024-06-06 10:29:36.010559] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 43 [2024-06-06 10:29:36.041507] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 44 [2024-06-06 10:29:36.041530] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501216-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601216-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701216-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 45 [2024-06-06 10:29:36.041563] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 46 [2024-06-06 10:29:36.041569] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501216-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601216-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701216-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 47 [2024-06-06 10:29:36.116656] INFO [TLOG.PARSER] stat_ignored_tps (ob_log_cluster_id_filter.cpp:130) [20201][][T0][Y0-0000000000000000-0-0] [lt=3] [TPS_STAT] CLUSTER_ID_IGNORED_PART_TPS=0.000 48 [2024-06-06 10:29:36.141667] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 49 [2024-06-06 10:29:36.141722] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 50 [2024-06-06 10:29:36.163442] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501217-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 51 [2024-06-06 10:29:36.163492] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501217-0-0] [lt=45] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 52 [2024-06-06 10:29:36.200050] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601217-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 53 [2024-06-06 10:29:36.200091] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601217-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 54 [2024-06-06 10:29:36.202263] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701217-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 55 [2024-06-06 10:29:36.202299] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701217-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 56 [2024-06-06 10:29:36.241822] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 57 [2024-06-06 10:29:36.241849] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501217-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601217-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701217-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 58 [2024-06-06 10:29:36.241884] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 59 [2024-06-06 10:29:36.241891] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501217-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601217-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701217-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 60 [2024-06-06 10:29:36.341987] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 61 [2024-06-06 10:29:36.342036] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 62 [2024-06-06 10:29:36.366838] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501218-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 63 [2024-06-06 10:29:36.366882] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501218-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 64 [2024-06-06 10:29:36.398102] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601218-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 65 [2024-06-06 10:29:36.398137] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601218-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 66 [2024-06-06 10:29:36.400064] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701218-0-0] [lt=26] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 67 [2024-06-06 10:29:36.400101] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701218-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 68 [2024-06-06 10:29:36.442141] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 69 [2024-06-06 10:29:36.442172] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501218-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601218-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701218-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 70 [2024-06-06 10:29:36.442210] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=37][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 71 [2024-06-06 10:29:36.442216] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501218-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601218-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701218-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 72 [2024-06-06 10:29:36.542317] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 73 [2024-06-06 10:29:36.542377] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=38] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 74 [2024-06-06 10:29:36.563936] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501219-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 75 [2024-06-06 10:29:36.563982] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501219-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 76 [2024-06-06 10:29:36.587263] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601219-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 77 [2024-06-06 10:29:36.587314] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601219-0-0] [lt=47] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 78 [2024-06-06 10:29:36.593717] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701219-0-0] [lt=25] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 79 [2024-06-06 10:29:36.593766] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701219-0-0] [lt=46] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 80 [2024-06-06 10:29:36.642487] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=29][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 81 [2024-06-06 10:29:36.642519] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501219-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601219-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701219-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 82 [2024-06-06 10:29:36.642556] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 83 [2024-06-06 10:29:36.642562] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501219-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601219-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701219-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 84 [2024-06-06 10:29:36.729400] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 85 [2024-06-06 10:29:36.729445] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 86 [2024-06-06 10:29:36.729855] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=29] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 87 [2024-06-06 10:29:36.730354] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 88 [2024-06-06 10:29:36.742657] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 89 [2024-06-06 10:29:36.742703] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 90 [2024-06-06 10:29:36.762919] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050121A-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 91 [2024-06-06 10:29:36.762961] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050121A-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 92 [2024-06-06 10:29:36.768171] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1717640976768170, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007614, add_bytes: 0 93 [2024-06-06 10:29:36.785069] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=24] PNIO [ratelimit] time: 1717640976785067, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007615, add_bytes: 0 94 [2024-06-06 10:29:36.791507] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060121A-0-0] [lt=29] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 95 [2024-06-06 10:29:36.791553] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060121A-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 96 [2024-06-06 10:29:36.793430] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070121A-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 97 [2024-06-06 10:29:36.793466] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070121A-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 98 [2024-06-06 10:29:36.842804] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 99 [2024-06-06 10:29:36.842829] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121A-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 100 [2024-06-06 10:29:36.842864] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 101 [2024-06-06 10:29:36.842871] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121A-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 102 [2024-06-06 10:29:36.942968] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 103 [2024-06-06 10:29:36.943032] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=46] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 104 [2024-06-06 10:29:36.969584] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=21] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 105 [2024-06-06 10:29:36.969700] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 106 [2024-06-06 10:29:36.969706] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=14] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 107 [2024-06-06 10:29:36.969712] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=21] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 108 [2024-06-06 10:29:36.971878] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050121B-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 109 [2024-06-06 10:29:36.971919] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050121B-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 110 [2024-06-06 10:29:36.997311] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060121B-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 111 [2024-06-06 10:29:36.997362] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060121B-0-0] [lt=47] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 112 [2024-06-06 10:29:36.999212] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070121B-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 113 [2024-06-06 10:29:36.999248] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070121B-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 114 [2024-06-06 10:29:37.008275] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=3477/0, request doing=0/0) 115 [2024-06-06 10:29:37.008641] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=3/0, request done=3476/0, request doing=0/0) 116 [2024-06-06 10:29:37.008920] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=3/0, request done=3476/0, request doing=0/0) 117 [2024-06-06 10:29:37.009356] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=3/0, request done=3476/0, request doing=0/0) 118 [2024-06-06 10:29:37.009869] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=20] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 119 [2024-06-06 10:29:37.010346] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 120 [2024-06-06 10:29:37.010350] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=12] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 121 [2024-06-06 10:29:37.010515] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 122 [2024-06-06 10:29:37.043128] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 123 [2024-06-06 10:29:37.043156] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121B-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 124 [2024-06-06 10:29:37.043201] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=44][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 125 [2024-06-06 10:29:37.043208] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121B-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 126 [2024-06-06 10:29:37.143307] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 127 [2024-06-06 10:29:37.143366] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=40] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 128 [2024-06-06 10:29:37.164396] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050121C-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 129 [2024-06-06 10:29:37.164442] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050121C-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 130 [2024-06-06 10:29:37.190307] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060121C-0-0] [lt=21] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 131 [2024-06-06 10:29:37.190354] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060121C-0-0] [lt=44] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 132 [2024-06-06 10:29:37.192114] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070121C-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 133 [2024-06-06 10:29:37.192148] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070121C-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 134 [2024-06-06 10:29:37.243465] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 135 [2024-06-06 10:29:37.243493] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121C-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 136 [2024-06-06 10:29:37.243533] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=38][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 137 [2024-06-06 10:29:37.243539] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121C-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 138 [2024-06-06 10:29:37.343635] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 139 [2024-06-06 10:29:37.343686] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 140 [2024-06-06 10:29:37.366103] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050121D-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 141 [2024-06-06 10:29:37.366156] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050121D-0-0] [lt=48] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 142 [2024-06-06 10:29:37.395742] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060121D-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 143 [2024-06-06 10:29:37.395781] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060121D-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 144 [2024-06-06 10:29:37.397513] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070121D-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 145 [2024-06-06 10:29:37.397550] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070121D-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 146 [2024-06-06 10:29:37.443783] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 147 [2024-06-06 10:29:37.443808] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121D-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 148 [2024-06-06 10:29:37.443845] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 149 [2024-06-06 10:29:37.443855] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=9][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121D-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 150 [2024-06-06 10:29:37.543952] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 151 [2024-06-06 10:29:37.544002] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 152 [2024-06-06 10:29:37.580212] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050121E-0-0] [lt=27] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 153 [2024-06-06 10:29:37.580262] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050121E-0-0] [lt=46] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 154 [2024-06-06 10:29:37.599555] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060121E-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 155 [2024-06-06 10:29:37.599597] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060121E-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 156 [2024-06-06 10:29:37.602023] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070121E-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 157 [2024-06-06 10:29:37.602062] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070121E-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 158 [2024-06-06 10:29:37.644097] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 159 [2024-06-06 10:29:37.644125] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121E-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 160 [2024-06-06 10:29:37.644160] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 161 [2024-06-06 10:29:37.644166] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121E-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 162 [2024-06-06 10:29:37.729359] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 163 [2024-06-06 10:29:37.729677] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 164 [2024-06-06 10:29:37.729685] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 165 [2024-06-06 10:29:37.729689] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 166 [2024-06-06 10:29:37.744289] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 167 [2024-06-06 10:29:37.744393] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=68] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 168 [2024-06-06 10:29:37.765662] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050121F-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 169 [2024-06-06 10:29:37.765748] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050121F-0-0] [lt=78] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 170 [2024-06-06 10:29:37.775814] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=22] PNIO [ratelimit] time: 1717640977775807, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007637, add_bytes: 0 171 [2024-06-06 10:29:37.792689] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=24] PNIO [ratelimit] time: 1717640977792686, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007619, add_bytes: 0 172 [2024-06-06 10:29:37.792646] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060121F-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 173 [2024-06-06 10:29:37.792741] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060121F-0-0] [lt=87] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 174 [2024-06-06 10:29:37.794674] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070121F-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 175 [2024-06-06 10:29:37.794707] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070121F-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 176 [2024-06-06 10:29:37.844508] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 177 [2024-06-06 10:29:37.844586] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=76][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121F-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 178 [2024-06-06 10:29:37.844648] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=61][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 179 [2024-06-06 10:29:37.844658] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=8][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050121F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060121F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070121F-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 180 [2024-06-06 10:29:37.944774] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 181 [2024-06-06 10:29:37.944821] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 182 [2024-06-06 10:29:37.969047] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 183 [2024-06-06 10:29:37.969370] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=22] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 184 [2024-06-06 10:29:37.969502] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 185 [2024-06-06 10:29:37.970480] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 186 [2024-06-06 10:29:37.980335] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501220-0-0] [lt=38] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 187 [2024-06-06 10:29:37.980399] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501220-0-0] [lt=58] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 188 [2024-06-06 10:29:37.999298] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601220-0-0] [lt=21] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 189 [2024-06-06 10:29:37.999353] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601220-0-0] [lt=50] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 190 [2024-06-06 10:29:38.008326] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=3/0, request done=3480/0, request doing=0/0) 191 [2024-06-06 10:29:38.008357] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=3/0, request done=3480/0, request doing=0/0) 192 [2024-06-06 10:29:38.008524] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=3/0, request done=3479/0, request doing=1/0) 193 [2024-06-06 10:29:38.008890] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=22] [RPC EASY STAT](log_str=conn count=3/0, request done=3480/0, request doing=0/0) 194 [2024-06-06 10:29:38.009345] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=12] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 195 [2024-06-06 10:29:38.009430] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701220-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 196 [2024-06-06 10:29:38.009459] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701220-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 197 [2024-06-06 10:29:38.009729] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=20] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 198 [2024-06-06 10:29:38.010348] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 199 [2024-06-06 10:29:38.010381] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=21] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 200 [2024-06-06 10:29:38.044914] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 201 [2024-06-06 10:29:38.044936] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501220-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601220-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701220-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 202 [2024-06-06 10:29:38.044966] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=29][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 203 [2024-06-06 10:29:38.044972] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501220-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601220-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701220-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 204 [2024-06-06 10:29:38.145074] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 205 [2024-06-06 10:29:38.145122] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=29] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 206 [2024-06-06 10:29:38.186646] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501221-0-0] [lt=23] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 207 [2024-06-06 10:29:38.186694] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501221-0-0] [lt=44] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 208 [2024-06-06 10:29:38.208724] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601221-0-0] [lt=21] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 209 [2024-06-06 10:29:38.208765] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601221-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 210 [2024-06-06 10:29:38.211019] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701221-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 211 [2024-06-06 10:29:38.211061] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701221-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 212 [2024-06-06 10:29:38.245218] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 213 [2024-06-06 10:29:38.245245] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501221-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601221-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701221-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 214 [2024-06-06 10:29:38.245277] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 215 [2024-06-06 10:29:38.245284] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501221-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601221-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701221-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 216 [2024-06-06 10:29:38.345358] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 217 [2024-06-06 10:29:38.345412] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 218 [2024-06-06 10:29:38.377563] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501222-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 219 [2024-06-06 10:29:38.377614] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501222-0-0] [lt=48] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 220 [2024-06-06 10:29:38.407641] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601222-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 221 [2024-06-06 10:29:38.407687] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601222-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 222 [2024-06-06 10:29:38.409596] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701222-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 223 [2024-06-06 10:29:38.409642] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701222-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 224 [2024-06-06 10:29:38.445497] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 225 [2024-06-06 10:29:38.445525] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501222-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601222-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701222-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 226 [2024-06-06 10:29:38.445561] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 227 [2024-06-06 10:29:38.445568] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501222-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601222-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701222-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 228 [2024-06-06 10:29:38.545665] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 229 [2024-06-06 10:29:38.545722] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=33] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 230 [2024-06-06 10:29:38.576699] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501223-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 231 [2024-06-06 10:29:38.576738] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501223-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 232 [2024-06-06 10:29:38.599778] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601223-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 233 [2024-06-06 10:29:38.599812] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601223-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 234 [2024-06-06 10:29:38.602241] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701223-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 235 [2024-06-06 10:29:38.602268] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701223-0-0] [lt=25] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 236 [2024-06-06 10:29:38.645829] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 237 [2024-06-06 10:29:38.645861] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501223-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601223-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701223-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 238 [2024-06-06 10:29:38.645911] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=48][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 239 [2024-06-06 10:29:38.645920] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=7][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501223-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601223-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701223-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 240 [2024-06-06 10:29:38.728734] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 241 [2024-06-06 10:29:38.728802] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 242 [2024-06-06 10:29:38.729720] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 243 [2024-06-06 10:29:38.730413] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=24] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 244 [2024-06-06 10:29:38.746028] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 245 [2024-06-06 10:29:38.746079] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 246 [2024-06-06 10:29:38.768787] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501224-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 247 [2024-06-06 10:29:38.768826] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501224-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 248 [2024-06-06 10:29:38.783410] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=65] PNIO [ratelimit] time: 1717640978783407, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007600, add_bytes: 0 249 [2024-06-06 10:29:38.800301] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=31] PNIO [ratelimit] time: 1717640978800299, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007613, add_bytes: 0 250 [2024-06-06 10:29:38.805595] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601224-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 251 [2024-06-06 10:29:38.805645] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601224-0-0] [lt=46] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 252 [2024-06-06 10:29:38.807145] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701224-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 253 [2024-06-06 10:29:38.807179] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701224-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 254 [2024-06-06 10:29:38.846177] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 255 [2024-06-06 10:29:38.846202] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501224-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601224-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701224-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 256 [2024-06-06 10:29:38.846239] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 257 [2024-06-06 10:29:38.846246] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501224-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601224-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701224-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 258 [2024-06-06 10:29:38.946352] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 259 [2024-06-06 10:29:38.946412] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=41] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 260 [2024-06-06 10:29:38.968620] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=22] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 261 [2024-06-06 10:29:38.968835] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 262 [2024-06-06 10:29:38.970048] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=24] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 263 [2024-06-06 10:29:38.970349] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 264 [2024-06-06 10:29:38.970400] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501225-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 265 [2024-06-06 10:29:38.970439] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501225-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 266 [2024-06-06 10:29:39.000410] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601225-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 267 [2024-06-06 10:29:39.000456] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601225-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 268 [2024-06-06 10:29:39.002460] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701225-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 269 [2024-06-06 10:29:39.002491] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701225-0-0] [lt=29] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 270 [2024-06-06 10:29:39.008448] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=3484/0, request doing=0/0) 271 [2024-06-06 10:29:39.008462] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=3484/0, request doing=0/0) 272 [2024-06-06 10:29:39.008513] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=26] [RPC EASY STAT](log_str=conn count=3/0, request done=3484/0, request doing=0/0) 273 [2024-06-06 10:29:39.009358] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=3/0, request done=3483/0, request doing=0/0) 274 [2024-06-06 10:29:39.009837] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 275 [2024-06-06 10:29:39.010376] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=16] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 276 [2024-06-06 10:29:39.010405] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=12] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 277 [2024-06-06 10:29:39.010750] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 278 [2024-06-06 10:29:39.046511] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 279 [2024-06-06 10:29:39.046540] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501225-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601225-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701225-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 280 [2024-06-06 10:29:39.046583] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=42][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 281 [2024-06-06 10:29:39.046590] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501225-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601225-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701225-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 282 [2024-06-06 10:29:39.146687] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 283 [2024-06-06 10:29:39.146736] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=33] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 284 [2024-06-06 10:29:39.186052] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501226-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 285 [2024-06-06 10:29:39.186106] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501226-0-0] [lt=50] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 286 [2024-06-06 10:29:39.207531] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601226-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 287 [2024-06-06 10:29:39.207573] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601226-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 288 [2024-06-06 10:29:39.209518] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701226-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 289 [2024-06-06 10:29:39.209546] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701226-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 290 [2024-06-06 10:29:39.246840] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 291 [2024-06-06 10:29:39.246871] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501226-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601226-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701226-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 292 [2024-06-06 10:29:39.246924] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=52][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 293 [2024-06-06 10:29:39.246930] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501226-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601226-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701226-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 294 [2024-06-06 10:29:39.301047] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=16] [STAT] [ALL_ZONE] INDEX=0/3 ZONE=az1 REGION=BJ STORAGE_TYPE=LOCAL VERSION=187 295 [2024-06-06 10:29:39.301072] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=20] [STAT] [ALL_ZONE] INDEX=1/3 ZONE=az2 REGION=BJ STORAGE_TYPE=LOCAL VERSION=187 296 [2024-06-06 10:29:39.301078] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_ZONE] INDEX=2/3 ZONE=az4 REGION=BJ STORAGE_TYPE=LOCAL VERSION=187 297 [2024-06-06 10:29:39.301083] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_ZONE] INDEX=0/3 ZONE=az1 ZONE_TYPE=ReadWrite VERSION=187 298 [2024-06-06 10:29:39.301087] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_ZONE] INDEX=1/3 ZONE=az2 ZONE_TYPE=ReadWrite VERSION=187 299 [2024-06-06 10:29:39.301091] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_ZONE] INDEX=2/3 ZONE=az4 ZONE_TYPE=ReadWrite VERSION=187 300 [2024-06-06 10:29:39.301094] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:454) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=2] [STAT] [ALL_ZONE] COUNT=3 VERSION=187 301 [2024-06-06 10:29:39.301102] INFO [TLOG] purge_stale_zone_records_ (ob_log_all_svr_cache.cpp:660) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=2] [STAT] [ALL_ZONE] [PURGE] PURGE_COUNT=0 CUR_COUNT=3 VERSION=187 302 [2024-06-06 10:29:39.305052] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_SERVER_LIST] INDEX=0/15 SERVER_ID=4 SERVER="10.50.54.157:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=187 303 [2024-06-06 10:29:39.305075] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=19] [STAT] [ALL_SERVER_LIST] INDEX=1/15 SERVER_ID=6 SERVER="10.50.54.245:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=187 304 [2024-06-06 10:29:39.305086] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=9] [STAT] [ALL_SERVER_LIST] INDEX=2/15 SERVER_ID=5 SERVER="10.50.56.27:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=187 305 [2024-06-06 10:29:39.305091] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_SERVER_LIST] INDEX=3/15 SERVER_ID=2 SERVER="10.50.56.28:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=187 306 [2024-06-06 10:29:39.305095] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=4/15 SERVER_ID=1 SERVER="10.50.73.94:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=187 307 [2024-06-06 10:29:39.305100] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=5/15 SERVER_ID=9 SERVER="10.52.18.156:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=187 308 [2024-06-06 10:29:39.305104] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=6/15 SERVER_ID=8 SERVER="10.52.18.158:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=187 309 [2024-06-06 10:29:39.305108] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=7/15 SERVER_ID=7 SERVER="10.52.18.180:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=187 310 [2024-06-06 10:29:39.305113] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=8/15 SERVER_ID=13 SERVER="10.53.0.225:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=187 311 [2024-06-06 10:29:39.305117] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=9/15 SERVER_ID=15 SERVER="10.53.73.35:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=187 312 [2024-06-06 10:29:39.305121] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=10/15 SERVER_ID=14 SERVER="10.53.73.37:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=187 313 [2024-06-06 10:29:39.305125] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=11/15 SERVER_ID=10 SERVER="10.88.10.155:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=187 314 [2024-06-06 10:29:39.305129] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=12/15 SERVER_ID=3 SERVER="10.88.10.156:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=187 315 [2024-06-06 10:29:39.305133] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=13/15 SERVER_ID=11 SERVER="10.88.20.97:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=187 316 [2024-06-06 10:29:39.305138] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=14/15 SERVER_ID=12 SERVER="10.88.8.206:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=187 317 [2024-06-06 10:29:39.305141] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:525) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=2] [STAT] [ALL_SERVER_LIST] COUNT=15 VERSION=187 318 [2024-06-06 10:29:39.305152] INFO [TLOG] purge_stale_records_ (ob_log_all_svr_cache.cpp:646) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=2] [STAT] [ALL_SERVER_LIST] [PURGE] PURGE_COUNT=0 CUR_COUNT=15 VERSION=187 319 [2024-06-06 10:29:39.330130] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=7] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24425242667}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}]) 320 [2024-06-06 10:29:39.330161] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=30] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24425242667}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24425242667}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}]) 321 [2024-06-06 10:29:39.330169] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=8] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24425242667}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24425242667}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}, {server:"10.88.8.206:2882", ranges:[LSN:{24291926016, 24425242667}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}]) 322 [2024-06-06 10:29:39.330186] INFO [TLOG] get_ls_svr_list_ (ob_log_route_service.cpp:809) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=7] get_ls_svr_list_ done(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}, svr_list={next_svr_index:0, svr_num:3, svr_items:[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24425242667}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.88.8.206:2882", ranges:[LSN:{24291926016, 24425242667}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24425242667}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}]}, tg_id_=259) 323 [2024-06-06 10:29:39.347024] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 324 [2024-06-06 10:29:39.347061] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=19] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 325 [2024-06-06 10:29:39.371043] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501227-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 326 [2024-06-06 10:29:39.371087] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501227-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 327 [2024-06-06 10:29:39.410799] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601227-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 328 [2024-06-06 10:29:39.410845] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601227-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 329 [2024-06-06 10:29:39.412908] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701227-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 330 [2024-06-06 10:29:39.412938] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701227-0-0] [lt=28] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 331 [2024-06-06 10:29:39.447154] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 332 [2024-06-06 10:29:39.447180] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501227-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601227-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701227-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 333 [2024-06-06 10:29:39.447222] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=41][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 334 [2024-06-06 10:29:39.447229] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501227-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601227-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701227-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 335 [2024-06-06 10:29:39.547326] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 336 [2024-06-06 10:29:39.547381] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=38] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 337 [2024-06-06 10:29:39.569855] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501228-0-0] [lt=24] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 338 [2024-06-06 10:29:39.569889] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501228-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 339 [2024-06-06 10:29:39.591992] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601228-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 340 [2024-06-06 10:29:39.592040] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601228-0-0] [lt=44] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 341 [2024-06-06 10:29:39.602184] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701228-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 342 [2024-06-06 10:29:39.602212] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701228-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 343 [2024-06-06 10:29:39.647482] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 344 [2024-06-06 10:29:39.647509] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501228-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601228-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701228-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 345 [2024-06-06 10:29:39.647546] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 346 [2024-06-06 10:29:39.647557] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=9][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501228-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601228-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701228-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 347 [2024-06-06 10:29:39.729046] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 348 [2024-06-06 10:29:39.729575] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 349 [2024-06-06 10:29:39.729756] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 350 [2024-06-06 10:29:39.730347] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=19] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 351 [2024-06-06 10:29:39.747651] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 352 [2024-06-06 10:29:39.747700] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 353 [2024-06-06 10:29:39.790863] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501229-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 354 [2024-06-06 10:29:39.790909] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501229-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 355 [2024-06-06 10:29:39.791026] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1717640979791024, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007617, add_bytes: 0 356 [2024-06-06 10:29:39.807918] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=27] PNIO [ratelimit] time: 1717640979807916, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007617, add_bytes: 0 357 [2024-06-06 10:29:39.810396] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601229-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 358 [2024-06-06 10:29:39.810438] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601229-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 359 [2024-06-06 10:29:39.812410] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701229-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 360 [2024-06-06 10:29:39.812438] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701229-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 361 [2024-06-06 10:29:39.847794] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 362 [2024-06-06 10:29:39.847813] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=19][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501229-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601229-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701229-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 363 [2024-06-06 10:29:39.847848] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 364 [2024-06-06 10:29:39.847854] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501229-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601229-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701229-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 365 [2024-06-06 10:29:39.947945] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 366 [2024-06-06 10:29:39.947991] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=28] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 367 [2024-06-06 10:29:39.968941] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=13] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 368 [2024-06-06 10:29:39.969150] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=20] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 369 [2024-06-06 10:29:39.969375] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=13] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 370 [2024-06-06 10:29:39.969586] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 371 [2024-06-06 10:29:39.974252] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050122A-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 372 [2024-06-06 10:29:39.974286] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050122A-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 373 [2024-06-06 10:29:39.996780] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060122A-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 374 [2024-06-06 10:29:39.996810] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060122A-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 375 [2024-06-06 10:29:39.998611] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070122A-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 376 [2024-06-06 10:29:39.998652] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070122A-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 377 [2024-06-06 10:29:40.008628] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=3487/0, request doing=0/0) 378 [2024-06-06 10:29:40.008672] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=3/0, request done=3488/0, request doing=0/0) 379 [2024-06-06 10:29:40.008829] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=3/0, request done=3488/0, request doing=0/0) 380 [2024-06-06 10:29:40.009311] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=3/0, request done=3487/0, request doing=0/0) 381 [2024-06-06 10:29:40.010358] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=9] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 382 [2024-06-06 10:29:40.010366] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=13] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 383 [2024-06-06 10:29:40.010406] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 384 [2024-06-06 10:29:40.010407] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 385 [2024-06-06 10:29:40.048092] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 386 [2024-06-06 10:29:40.048123] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122A-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 387 [2024-06-06 10:29:40.048160] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 388 [2024-06-06 10:29:40.048167] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122A-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 389 [2024-06-06 10:29:40.148268] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 390 [2024-06-06 10:29:40.148315] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 391 [2024-06-06 10:29:40.225716] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050122B-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 392 [2024-06-06 10:29:40.225755] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050122B-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 393 [2024-06-06 10:29:40.245941] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060122B-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 394 [2024-06-06 10:29:40.245978] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060122B-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 395 [2024-06-06 10:29:40.247917] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070122B-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 396 [2024-06-06 10:29:40.247963] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070122B-0-0] [lt=43] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 397 [2024-06-06 10:29:40.248422] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 398 [2024-06-06 10:29:40.248447] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122B-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 399 [2024-06-06 10:29:40.248479] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 400 [2024-06-06 10:29:40.248486] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122B-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 401 [2024-06-06 10:29:40.348585] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 402 [2024-06-06 10:29:40.348635] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 403 [2024-06-06 10:29:40.377171] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050122C-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 404 [2024-06-06 10:29:40.377211] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050122C-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 405 [2024-06-06 10:29:40.403323] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060122C-0-0] [lt=19] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 406 [2024-06-06 10:29:40.403366] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060122C-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 407 [2024-06-06 10:29:40.405283] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070122C-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 408 [2024-06-06 10:29:40.405352] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070122C-0-0] [lt=45] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 409 [2024-06-06 10:29:40.448731] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 410 [2024-06-06 10:29:40.448758] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122C-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 411 [2024-06-06 10:29:40.448794] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 412 [2024-06-06 10:29:40.448800] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122C-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 413 [2024-06-06 10:29:40.548898] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 414 [2024-06-06 10:29:40.548949] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 415 [2024-06-06 10:29:40.575814] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050122D-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 416 [2024-06-06 10:29:40.575853] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050122D-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 417 [2024-06-06 10:29:40.598265] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060122D-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 418 [2024-06-06 10:29:40.598315] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060122D-0-0] [lt=47] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 419 [2024-06-06 10:29:40.603109] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070122D-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 420 [2024-06-06 10:29:40.603158] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070122D-0-0] [lt=46] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 421 [2024-06-06 10:29:40.649047] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 422 [2024-06-06 10:29:40.649078] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122D-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 423 [2024-06-06 10:29:40.649123] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=44][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 424 [2024-06-06 10:29:40.649130] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122D-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 425 [2024-06-06 10:29:40.729389] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=23] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 426 [2024-06-06 10:29:40.729513] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 427 [2024-06-06 10:29:40.729612] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 428 [2024-06-06 10:29:40.730400] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 429 [2024-06-06 10:29:40.749227] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 430 [2024-06-06 10:29:40.749271] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 431 [2024-06-06 10:29:40.773017] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050122E-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 432 [2024-06-06 10:29:40.773055] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050122E-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 433 [2024-06-06 10:29:40.798638] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=22] PNIO [ratelimit] time: 1717640980798635, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007611, add_bytes: 0 434 [2024-06-06 10:29:40.798670] INFO eloop_run (eloop.c:136) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=26] PNIO print mem cache info mem_freelists_str:0,0,0,0,0,0,0,0,0,, mod_report:DEFAULT:25,249,088 435 [2024-06-06 10:29:40.798911] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060122E-0-0] [lt=24] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 436 [2024-06-06 10:29:40.798939] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060122E-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 437 [2024-06-06 10:29:40.801048] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070122E-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 438 [2024-06-06 10:29:40.801083] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070122E-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 439 [2024-06-06 10:29:40.815536] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=23] PNIO [ratelimit] time: 1717640980815534, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007618, add_bytes: 0 440 [2024-06-06 10:29:40.849365] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 441 [2024-06-06 10:29:40.849392] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122E-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 442 [2024-06-06 10:29:40.849427] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 443 [2024-06-06 10:29:40.849433] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122E-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 444 [2024-06-06 10:29:40.949534] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 445 [2024-06-06 10:29:40.949588] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=37] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 446 [2024-06-06 10:29:40.969827] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 447 [2024-06-06 10:29:40.969980] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=21] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 448 [2024-06-06 10:29:40.970185] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=19] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 449 [2024-06-06 10:29:40.970354] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 450 [2024-06-06 10:29:40.980550] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050122F-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 451 [2024-06-06 10:29:40.980588] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050122F-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 452 [2024-06-06 10:29:41.001100] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060122F-0-0] [lt=19] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 453 [2024-06-06 10:29:41.001126] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060122F-0-0] [lt=25] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 454 [2024-06-06 10:29:41.003154] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070122F-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 455 [2024-06-06 10:29:41.003193] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070122F-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 456 [2024-06-06 10:29:41.008204] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=3/0, request done=3492/0, request doing=0/0) 457 [2024-06-06 10:29:41.008595] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=23] [RPC EASY STAT](log_str=conn count=3/0, request done=3491/0, request doing=0/0) 458 [2024-06-06 10:29:41.009156] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=27] [RPC EASY STAT](log_str=conn count=3/0, request done=3491/0, request doing=0/0) 459 [2024-06-06 10:29:41.009185] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=3491/0, request doing=0/0) 460 [2024-06-06 10:29:41.010353] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=22] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 461 [2024-06-06 10:29:41.010356] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 462 [2024-06-06 10:29:41.010404] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=20] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 463 [2024-06-06 10:29:41.010780] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 464 [2024-06-06 10:29:41.049682] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 465 [2024-06-06 10:29:41.049707] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122F-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 466 [2024-06-06 10:29:41.049746] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=38][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 467 [2024-06-06 10:29:41.049752] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050122F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060122F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070122F-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 468 [2024-06-06 10:29:41.149847] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 469 [2024-06-06 10:29:41.149897] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 470 [2024-06-06 10:29:41.176915] INFO [TLOG] add_ls_and_fetch_until_the_progress_is_reached (ob_log_meta_data_fetcher.cpp:195) [19959][][T0][Y0-0000000000000000-0-0] [lt=6] ObLogFetcher check_progress is less than end_timestamp(tenant_id=1020, cur_progress=1717577324520923022, end_tstamp_ns=1717603207000000000) 471 [2024-06-06 10:29:41.176957] INFO [TLOG] print_stat_info (ob_log_task_pool.h:266) [19959][][T0][Y0-0000000000000000-0-0] [lt=38] [STAT] [TRANS_TASK_POOL] OUT=0/1000 FIXED=0/1000 DYNAMIC=0 PAGE_POOL=1000/1000 472 [2024-06-06 10:29:41.182784] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501230-0-0] [lt=23] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 473 [2024-06-06 10:29:41.182825] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501230-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 474 [2024-06-06 10:29:41.207527] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601230-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 475 [2024-06-06 10:29:41.207568] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601230-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 476 [2024-06-06 10:29:41.210142] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701230-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 477 [2024-06-06 10:29:41.210205] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701230-0-0] [lt=57] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 478 [2024-06-06 10:29:41.250027] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 479 [2024-06-06 10:29:41.250088] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=59][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501230-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601230-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701230-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 480 [2024-06-06 10:29:41.250144] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=55][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 481 [2024-06-06 10:29:41.250151] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501230-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601230-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701230-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 482 [2024-06-06 10:29:41.350250] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 483 [2024-06-06 10:29:41.350300] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 484 [2024-06-06 10:29:41.390551] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501231-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 485 [2024-06-06 10:29:41.390589] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501231-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 486 [2024-06-06 10:29:41.409381] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601231-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 487 [2024-06-06 10:29:41.409420] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601231-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 488 [2024-06-06 10:29:41.411464] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701231-0-0] [lt=29] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 489 [2024-06-06 10:29:41.411508] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701231-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 490 [2024-06-06 10:29:41.450411] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 491 [2024-06-06 10:29:41.450445] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501231-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601231-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701231-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 492 [2024-06-06 10:29:41.450483] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=37][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 493 [2024-06-06 10:29:41.450489] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501231-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601231-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701231-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 494 [2024-06-06 10:29:41.550585] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 495 [2024-06-06 10:29:41.550630] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 496 [2024-06-06 10:29:41.590826] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501232-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 497 [2024-06-06 10:29:41.590863] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501232-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 498 [2024-06-06 10:29:41.609924] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601232-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 499 [2024-06-06 10:29:41.609959] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601232-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 500 [2024-06-06 10:29:41.611945] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701232-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 501 [2024-06-06 10:29:41.611989] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701232-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 502 [2024-06-06 10:29:41.650732] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=29][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 503 [2024-06-06 10:29:41.650756] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=23][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501232-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601232-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701232-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 504 [2024-06-06 10:29:41.650793] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 505 [2024-06-06 10:29:41.650799] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501232-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601232-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701232-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 506 [2024-06-06 10:29:41.729066] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 507 [2024-06-06 10:29:41.729644] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 508 [2024-06-06 10:29:41.729765] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=26] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 509 [2024-06-06 10:29:41.730432] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=24] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 510 [2024-06-06 10:29:41.750894] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 511 [2024-06-06 10:29:41.750938] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 512 [2024-06-06 10:29:41.781633] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501233-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 513 [2024-06-06 10:29:41.781674] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501233-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 514 [2024-06-06 10:29:41.802615] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601233-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 515 [2024-06-06 10:29:41.802646] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601233-0-0] [lt=29] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 516 [2024-06-06 10:29:41.804453] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701233-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 517 [2024-06-06 10:29:41.804496] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701233-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 518 [2024-06-06 10:29:41.806251] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=4] PNIO [ratelimit] time: 1717640981806249, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007614, add_bytes: 0 519 [2024-06-06 10:29:41.823150] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1717640981823149, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007615, add_bytes: 0 520 [2024-06-06 10:29:41.851040] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 521 [2024-06-06 10:29:41.851067] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501233-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601233-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701233-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 522 [2024-06-06 10:29:41.851104] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 523 [2024-06-06 10:29:41.851110] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501233-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601233-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701233-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 524 [2024-06-06 10:29:41.951205] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 525 [2024-06-06 10:29:41.951252] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 526 [2024-06-06 10:29:41.968894] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=26] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 527 [2024-06-06 10:29:41.969348] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 528 [2024-06-06 10:29:41.969492] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=13] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 529 [2024-06-06 10:29:41.970390] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 530 [2024-06-06 10:29:41.981962] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501234-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 531 [2024-06-06 10:29:41.981995] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501234-0-0] [lt=30] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 532 [2024-06-06 10:29:42.000418] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601234-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 533 [2024-06-06 10:29:42.000448] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601234-0-0] [lt=28] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 534 [2024-06-06 10:29:42.002508] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701234-0-0] [lt=29] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 535 [2024-06-06 10:29:42.002552] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701234-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 536 [2024-06-06 10:29:42.008481] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=3/0, request done=3495/0, request doing=0/0) 537 [2024-06-06 10:29:42.008560] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=3/0, request done=3495/0, request doing=0/0) 538 [2024-06-06 10:29:42.008651] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=3/0, request done=3495/0, request doing=0/0) 539 [2024-06-06 10:29:42.009026] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=3/0, request done=3495/0, request doing=0/0) 540 [2024-06-06 10:29:42.009654] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 541 [2024-06-06 10:29:42.009818] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 542 [2024-06-06 10:29:42.010340] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=17] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 543 [2024-06-06 10:29:42.010344] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=16] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 544 [2024-06-06 10:29:42.051359] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 545 [2024-06-06 10:29:42.051385] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501234-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601234-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701234-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 546 [2024-06-06 10:29:42.051426] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=40][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 547 [2024-06-06 10:29:42.051432] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=4][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501234-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601234-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701234-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 548 [2024-06-06 10:29:42.117050] INFO [TLOG.FETCHER] print_stat (ob_log_fetch_stream_container_mgr.cpp:181) [20201][][T0][Y0-0000000000000000-0-0] [lt=27] [STAT] [FS_CONTAINER_POOL] USED=1 FREE=0 FIXED=16 DYNAMIC=0 549 [2024-06-06 10:29:42.117073] INFO [TLOG.FETCHER] print_stat (ob_log_fetch_stream_pool.cpp:76) [20201][][T0][Y0-0000000000000000-0-0] [lt=22] [STAT] [FETCH_STREAM_POOL] USED=0 FREE=0 FIXED=16 DYNAMIC=0 550 [2024-06-06 10:29:42.117079] INFO [TLOG.FETCHER] print_stat (ob_log_fetch_log_rpc.cpp:1400) [20201][][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [RPC_RESULT_POOL] USED=0 FREE=0 FIXED=16 DYNAMIC=0 551 [2024-06-06 10:29:42.117114] INFO [TLOG.FETCHER] operator() (ob_log_fetch_stream_container_mgr.cpp:230) [20201][][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] [FETCH_STREAM] TENANT=1020, TRAFFIC=0.00B/sec 552 [2024-06-06 10:29:42.117141] INFO [TLOG.FETCHER] print_fetcher_stat_ (ob_log_fetcher.cpp:840) [20201][][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [FETCHER](upper_limit="[2024-06-05 16:49:14.520923]", dml_progress_limit_sec=30, fetcher_delay="[17:40:57.596209]") 553 [2024-06-06 10:29:42.117216] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:429) [20201][][T0][Y0-0000000000000000-0-0] [lt=7] [STAT] slow fetch progress start. ls_count=1/1 554 [2024-06-06 10:29:42.117230] INFO [TLOG.FETCHER] print_fetch_progress (ob_log_ls_fetch_ctx.cpp:1454) [20201][][T0][Y0-0000000000000000-0-0] [lt=5] [STAT] slow fetch progress idx=0/1 tls_id={tenant_id:1020, ls_id:{id:1}} mod=IDLE_POOL discarded=0 delay=[17:40:57.596293] tps=0.00 progress={next_lsn:{lsn:18446744073709551615}, log_progress:"[2024-06-05 16:48:44.520923]", log_touch_tstamp:"[2024-06-06 10:14:01.120965]"} 555 [2024-06-06 10:29:42.117237] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:436) [20201][][T0][Y0-0000000000000000-0-0] [lt=6] [STAT] slow fetch progress end 556 [2024-06-06 10:29:42.117241] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:441) [20201][][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] slow dispatch progress start. ls_count=1/1 557 [2024-06-06 10:29:42.117250] INFO [TLOG.FETCHER] print_dispatch_progress (ob_log_ls_fetch_ctx.cpp:1476) [20201][][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] slow dispatch progress idx=0/1 tls_id={tenant_id:1020, ls_id:{id:1}} delay=[17:40:57.596293] pending_task(queue/total)=0/0 dispatch_progress=[2024-06-05 16:48:44.520923] last_dispatch_log_lsn=18446744073709551615 next_task=INVALID next_trans(log_lsn=18446744073709551615,committed=0,ready_to_commit=0,global_version=[1970-01-01 08:00:00.000000]) checkpoint=[1970-01-01 08:00:00.000000] 558 [2024-06-06 10:29:42.117261] INFO [TLOG.FETCHER] print_k_slowest_ls (ob_log_ls_fetch_mgr.cpp:448) [20201][][T0][Y0-0000000000000000-0-0] [lt=11] [STAT] slow dispatch progress end 559 [2024-06-06 10:29:42.151531] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 560 [2024-06-06 10:29:42.151581] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 561 [2024-06-06 10:29:42.179852] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501235-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 562 [2024-06-06 10:29:42.179890] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501235-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 563 [2024-06-06 10:29:42.202429] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601235-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 564 [2024-06-06 10:29:42.202466] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601235-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 565 [2024-06-06 10:29:42.207317] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701235-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 566 [2024-06-06 10:29:42.207370] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701235-0-0] [lt=49] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 567 [2024-06-06 10:29:42.251678] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=28][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 568 [2024-06-06 10:29:42.251706] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501235-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601235-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701235-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 569 [2024-06-06 10:29:42.251740] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 570 [2024-06-06 10:29:42.251747] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501235-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601235-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701235-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 571 [2024-06-06 10:29:42.351849] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 572 [2024-06-06 10:29:42.351902] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 573 [2024-06-06 10:29:42.375083] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501236-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 574 [2024-06-06 10:29:42.375122] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501236-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 575 [2024-06-06 10:29:42.400560] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601236-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 576 [2024-06-06 10:29:42.400596] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601236-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 577 [2024-06-06 10:29:42.402344] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701236-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 578 [2024-06-06 10:29:42.402393] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701236-0-0] [lt=46] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 579 [2024-06-06 10:29:42.451998] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 580 [2024-06-06 10:29:42.452024] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501236-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601236-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701236-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 581 [2024-06-06 10:29:42.452059] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 582 [2024-06-06 10:29:42.452067] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501236-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601236-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701236-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 583 [2024-06-06 10:29:42.552168] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 584 [2024-06-06 10:29:42.552219] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=35] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 585 [2024-06-06 10:29:42.590641] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501237-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 586 [2024-06-06 10:29:42.590680] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501237-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 587 [2024-06-06 10:29:42.614190] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601237-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 588 [2024-06-06 10:29:42.614229] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601237-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 589 [2024-06-06 10:29:42.615892] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701237-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 590 [2024-06-06 10:29:42.615933] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701237-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 591 [2024-06-06 10:29:42.652317] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 592 [2024-06-06 10:29:42.652348] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501237-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601237-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701237-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 593 [2024-06-06 10:29:42.652384] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 594 [2024-06-06 10:29:42.652390] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501237-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601237-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701237-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 595 [2024-06-06 10:29:42.729579] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 596 [2024-06-06 10:29:42.729683] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 597 [2024-06-06 10:29:42.730107] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=20] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 598 [2024-06-06 10:29:42.730377] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 599 [2024-06-06 10:29:42.752492] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 600 [2024-06-06 10:29:42.752549] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=35] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 601 [2024-06-06 10:29:42.778445] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501238-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 602 [2024-06-06 10:29:42.778504] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501238-0-0] [lt=54] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 603 [2024-06-06 10:29:42.797125] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601238-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 604 [2024-06-06 10:29:42.797175] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601238-0-0] [lt=46] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 605 [2024-06-06 10:29:42.798884] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701238-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 606 [2024-06-06 10:29:42.798923] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701238-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 607 [2024-06-06 10:29:42.813872] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=18] PNIO [ratelimit] time: 1717640982813869, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007620, add_bytes: 0 608 [2024-06-06 10:29:42.830767] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=22] PNIO [ratelimit] time: 1717640982830765, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007616, add_bytes: 0 609 [2024-06-06 10:29:42.852658] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 610 [2024-06-06 10:29:42.852686] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501238-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601238-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701238-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 611 [2024-06-06 10:29:42.852722] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 612 [2024-06-06 10:29:42.852728] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501238-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601238-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701238-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 613 [2024-06-06 10:29:42.952828] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 614 [2024-06-06 10:29:42.952886] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=41] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 615 [2024-06-06 10:29:42.968416] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=17] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 616 [2024-06-06 10:29:42.968578] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=11] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 617 [2024-06-06 10:29:42.969793] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=19] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 618 [2024-06-06 10:29:42.970195] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 619 [2024-06-06 10:29:42.992081] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501239-0-0] [lt=22] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 620 [2024-06-06 10:29:42.992116] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501239-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 621 [2024-06-06 10:29:43.008301] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=3498/0, request doing=1/0) 622 [2024-06-06 10:29:43.009107] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=3/0, request done=3498/0, request doing=0/0) 623 [2024-06-06 10:29:43.009134] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=3/0, request done=3499/0, request doing=0/0) 624 [2024-06-06 10:29:43.009337] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=3/0, request done=3498/0, request doing=0/0) 625 [2024-06-06 10:29:43.010350] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 626 [2024-06-06 10:29:43.010370] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 627 [2024-06-06 10:29:43.010370] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=16] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 628 [2024-06-06 10:29:43.010849] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 629 [2024-06-06 10:29:43.013861] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601239-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 630 [2024-06-06 10:29:43.013918] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601239-0-0] [lt=53] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 631 [2024-06-06 10:29:43.015663] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701239-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 632 [2024-06-06 10:29:43.015690] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701239-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 633 [2024-06-06 10:29:43.052993] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 634 [2024-06-06 10:29:43.053023] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=29][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501239-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601239-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701239-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 635 [2024-06-06 10:29:43.053067] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=44][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 636 [2024-06-06 10:29:43.053074] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501239-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601239-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701239-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 637 [2024-06-06 10:29:43.153171] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 638 [2024-06-06 10:29:43.153220] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 639 [2024-06-06 10:29:43.179024] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050123A-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 640 [2024-06-06 10:29:43.179067] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050123A-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 641 [2024-06-06 10:29:43.201357] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060123A-0-0] [lt=23] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 642 [2024-06-06 10:29:43.201402] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060123A-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 643 [2024-06-06 10:29:43.203138] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070123A-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 644 [2024-06-06 10:29:43.203168] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070123A-0-0] [lt=28] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 645 [2024-06-06 10:29:43.253316] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 646 [2024-06-06 10:29:43.253347] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123A-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 647 [2024-06-06 10:29:43.253389] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=41][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 648 [2024-06-06 10:29:43.253395] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123A-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 649 [2024-06-06 10:29:43.277477] INFO [COMMON] handle (memory_dump.cpp:468) [19994][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=13] handle dump task(task={type_:2, dump_all_:false, p_context_:null, slot_idx_:0, dump_tenant_ctx_:false, tenant_id_:0, ctx_id_:0, p_chunk_:null}) 650 [2024-06-06 10:29:43.278871] INFO handle (memory_dump.cpp:589) [19994][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=27] statistics: 651 tenant_cnt: 3, max_chunk_cnt: 524288 652 tenant_id ctx_id chunk_cnt label_cnt segv_cnt 653 1 0 2 4 0 654 500 0 44 95 0 655 500 8 13 1 0 656 500 17 9 1 0 657 500 23 1 1 0 658 cost_time: 1279 659 [2024-06-06 10:29:43.353494] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 660 [2024-06-06 10:29:43.353543] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 661 [2024-06-06 10:29:43.395776] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050123B-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 662 [2024-06-06 10:29:43.395818] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050123B-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 663 [2024-06-06 10:29:43.415039] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060123B-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 664 [2024-06-06 10:29:43.415081] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060123B-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 665 [2024-06-06 10:29:43.417006] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070123B-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 666 [2024-06-06 10:29:43.417034] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070123B-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 667 [2024-06-06 10:29:43.453646] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 668 [2024-06-06 10:29:43.453678] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123B-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 669 [2024-06-06 10:29:43.453714] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 670 [2024-06-06 10:29:43.453720] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=4][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123B-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 671 [2024-06-06 10:29:43.553816] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 672 [2024-06-06 10:29:43.553861] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 673 [2024-06-06 10:29:43.574823] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050123C-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 674 [2024-06-06 10:29:43.574864] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050123C-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 675 [2024-06-06 10:29:43.594821] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060123C-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 676 [2024-06-06 10:29:43.594862] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060123C-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 677 [2024-06-06 10:29:43.597026] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070123C-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 678 [2024-06-06 10:29:43.597057] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070123C-0-0] [lt=30] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 679 [2024-06-06 10:29:43.653957] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 680 [2024-06-06 10:29:43.653983] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123C-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 681 [2024-06-06 10:29:43.654022] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=38][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 682 [2024-06-06 10:29:43.654029] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123C-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 683 [2024-06-06 10:29:43.729354] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 684 [2024-06-06 10:29:43.729720] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 685 [2024-06-06 10:29:43.729943] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=24] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 686 [2024-06-06 10:29:43.730354] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 687 [2024-06-06 10:29:43.754127] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 688 [2024-06-06 10:29:43.754172] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 689 [2024-06-06 10:29:43.803338] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050123D-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 690 [2024-06-06 10:29:43.803374] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050123D-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 691 [2024-06-06 10:29:43.821487] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=23] PNIO [ratelimit] time: 1717640983821484, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007615, add_bytes: 0 692 [2024-06-06 10:29:43.822216] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060123D-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 693 [2024-06-06 10:29:43.822246] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060123D-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 694 [2024-06-06 10:29:43.823887] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070123D-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 695 [2024-06-06 10:29:43.823916] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070123D-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 696 [2024-06-06 10:29:43.838384] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=23] PNIO [ratelimit] time: 1717640983838383, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007618, add_bytes: 0 697 [2024-06-06 10:29:43.854271] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 698 [2024-06-06 10:29:43.854293] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123D-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 699 [2024-06-06 10:29:43.854324] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 700 [2024-06-06 10:29:43.854336] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=10][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123D-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 701 [2024-06-06 10:29:43.954434] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=23] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 702 [2024-06-06 10:29:43.954486] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 703 [2024-06-06 10:29:43.968613] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=21] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 704 [2024-06-06 10:29:43.969246] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=15] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 705 [2024-06-06 10:29:43.969350] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 706 [2024-06-06 10:29:43.970011] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 707 [2024-06-06 10:29:43.977480] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050123E-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 708 [2024-06-06 10:29:43.977515] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050123E-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 709 [2024-06-06 10:29:43.996714] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060123E-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 710 [2024-06-06 10:29:43.996744] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060123E-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 711 [2024-06-06 10:29:43.998654] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070123E-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 712 [2024-06-06 10:29:43.998697] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070123E-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 713 [2024-06-06 10:29:44.008529] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=3/0, request done=3502/0, request doing=0/0) 714 [2024-06-06 10:29:44.008710] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=25] [RPC EASY STAT](log_str=conn count=3/0, request done=3503/0, request doing=0/0) 715 [2024-06-06 10:29:44.008734] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=3/0, request done=3503/0, request doing=0/0) 716 [2024-06-06 10:29:44.009079] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=3/0, request done=3502/0, request doing=0/0) 717 [2024-06-06 10:29:44.009718] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 718 [2024-06-06 10:29:44.009968] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=20] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 719 [2024-06-06 10:29:44.010015] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 720 [2024-06-06 10:29:44.010085] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=15] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 721 [2024-06-06 10:29:44.054593] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 722 [2024-06-06 10:29:44.054620] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123E-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 723 [2024-06-06 10:29:44.054663] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=42][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 724 [2024-06-06 10:29:44.054671] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=7][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123E-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 725 [2024-06-06 10:29:44.154771] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=23] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 726 [2024-06-06 10:29:44.154820] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 727 [2024-06-06 10:29:44.187216] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050123F-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 728 [2024-06-06 10:29:44.187255] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050123F-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 729 [2024-06-06 10:29:44.206311] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060123F-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 730 [2024-06-06 10:29:44.206360] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060123F-0-0] [lt=46] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 731 [2024-06-06 10:29:44.211503] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070123F-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 732 [2024-06-06 10:29:44.211537] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070123F-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 733 [2024-06-06 10:29:44.254922] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 734 [2024-06-06 10:29:44.254951] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123F-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 735 [2024-06-06 10:29:44.254989] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=38][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 736 [2024-06-06 10:29:44.255000] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=10][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050123F-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060123F-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070123F-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 737 [2024-06-06 10:29:44.337178] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=19] [STAT] [ALL_ZONE] INDEX=0/3 ZONE=az1 REGION=BJ STORAGE_TYPE=LOCAL VERSION=188 738 [2024-06-06 10:29:44.337201] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=20] [STAT] [ALL_ZONE] INDEX=1/3 ZONE=az2 REGION=BJ STORAGE_TYPE=LOCAL VERSION=188 739 [2024-06-06 10:29:44.337206] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:425) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_ZONE] INDEX=2/3 ZONE=az4 REGION=BJ STORAGE_TYPE=LOCAL VERSION=188 740 [2024-06-06 10:29:44.337212] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_ZONE] INDEX=0/3 ZONE=az1 ZONE_TYPE=ReadWrite VERSION=188 741 [2024-06-06 10:29:44.337216] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_ZONE] INDEX=1/3 ZONE=az2 ZONE_TYPE=ReadWrite VERSION=188 742 [2024-06-06 10:29:44.337220] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:450) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_ZONE] INDEX=2/3 ZONE=az4 ZONE_TYPE=ReadWrite VERSION=188 743 [2024-06-06 10:29:44.337223] INFO [TLOG] update_zone_cache_ (ob_log_all_svr_cache.cpp:454) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_ZONE] COUNT=3 VERSION=188 744 [2024-06-06 10:29:44.337232] INFO [TLOG] purge_stale_zone_records_ (ob_log_all_svr_cache.cpp:660) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_ZONE] [PURGE] PURGE_COUNT=0 CUR_COUNT=3 VERSION=188 745 [2024-06-06 10:29:44.341917] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_SERVER_LIST] INDEX=0/15 SERVER_ID=4 SERVER="10.50.54.157:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=188 746 [2024-06-06 10:29:44.341933] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=14] [STAT] [ALL_SERVER_LIST] INDEX=1/15 SERVER_ID=6 SERVER="10.50.54.245:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=188 747 [2024-06-06 10:29:44.341938] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_SERVER_LIST] INDEX=2/15 SERVER_ID=5 SERVER="10.50.56.27:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=188 748 [2024-06-06 10:29:44.341944] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] [STAT] [ALL_SERVER_LIST] INDEX=3/15 SERVER_ID=2 SERVER="10.50.56.28:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=188 749 [2024-06-06 10:29:44.341954] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=9] [STAT] [ALL_SERVER_LIST] INDEX=4/15 SERVER_ID=1 SERVER="10.50.73.94:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=188 750 [2024-06-06 10:29:44.341958] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=5/15 SERVER_ID=9 SERVER="10.52.18.156:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=188 751 [2024-06-06 10:29:44.341962] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=6/15 SERVER_ID=8 SERVER="10.52.18.158:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=188 752 [2024-06-06 10:29:44.341967] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=7/15 SERVER_ID=7 SERVER="10.52.18.180:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=188 753 [2024-06-06 10:29:44.341971] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=8/15 SERVER_ID=13 SERVER="10.53.0.225:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=188 754 [2024-06-06 10:29:44.341975] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=9/15 SERVER_ID=15 SERVER="10.53.73.35:2882" STATUS=1(ACTIVE) ZONE=az1 REGION=BJ(LOW) VERSION=188 755 [2024-06-06 10:29:44.341979] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=10/15 SERVER_ID=14 SERVER="10.53.73.37:2882" STATUS=1(ACTIVE) ZONE=az2 REGION=BJ(LOW) VERSION=188 756 [2024-06-06 10:29:44.341983] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=11/15 SERVER_ID=10 SERVER="10.88.10.155:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=188 757 [2024-06-06 10:29:44.341987] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=12/15 SERVER_ID=3 SERVER="10.88.10.156:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=188 758 [2024-06-06 10:29:44.341991] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=13/15 SERVER_ID=11 SERVER="10.88.20.97:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=188 759 [2024-06-06 10:29:44.341996] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:521) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=3] [STAT] [ALL_SERVER_LIST] INDEX=14/15 SERVER_ID=12 SERVER="10.88.8.206:2882" STATUS=1(ACTIVE) ZONE=az4 REGION=BJ(LOW) VERSION=188 760 [2024-06-06 10:29:44.341999] INFO [TLOG] update_server_cache_ (ob_log_all_svr_cache.cpp:525) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=2] [STAT] [ALL_SERVER_LIST] COUNT=15 VERSION=188 761 [2024-06-06 10:29:44.342007] INFO [TLOG] purge_stale_records_ (ob_log_all_svr_cache.cpp:646) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=2] [STAT] [ALL_SERVER_LIST] [PURGE] PURGE_COUNT=0 CUR_COUNT=15 VERSION=188 762 [2024-06-06 10:29:44.355094] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 763 [2024-06-06 10:29:44.355141] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 764 [2024-06-06 10:29:44.381035] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=4] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24425252778}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}]) 765 [2024-06-06 10:29:44.381060] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=23] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24425252778}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24425252778}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}]) 766 [2024-06-06 10:29:44.381068] INFO [TLOG.FETCHER] add_server_or_update (ob_ls_server_list.cpp:87) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=7] add_server_or_update(next_svr_index=0, svr_items=[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24425252778}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24425252778}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}, {server:"10.88.8.206:2882", ranges:[LSN:{24291926016, 24425252778}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}]) 767 [2024-06-06 10:29:44.381079] INFO [TLOG] get_ls_svr_list_ (ob_log_route_service.cpp:809) [20168][LogRouter][T0][Y0-0000000000000000-0-0] [lt=7] get_ls_svr_list_ done(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}, svr_list={next_svr_index:0, svr_num:3, svr_items:[{server:"10.50.54.157:2882", ranges:[LSN:{24291926016, 24425252778}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.88.8.206:2882", ranges:[LSN:{24291926016, 24425252778}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:0]}, {server:"10.52.18.156:2882", ranges:[LSN:{24291926016, 24425252778}], priority:[is_meta_table:1 region:LOW, replica:FULL, is_leader:1]}]}, tg_id_=259) 768 [2024-06-06 10:29:44.389476] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501240-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 769 [2024-06-06 10:29:44.389512] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501240-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 770 [2024-06-06 10:29:44.407956] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601240-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 771 [2024-06-06 10:29:44.407996] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601240-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 772 [2024-06-06 10:29:44.415376] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701240-0-0] [lt=18] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 773 [2024-06-06 10:29:44.415406] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701240-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 774 [2024-06-06 10:29:44.455247] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 775 [2024-06-06 10:29:44.455270] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501240-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601240-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701240-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 776 [2024-06-06 10:29:44.455306] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=35][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 777 [2024-06-06 10:29:44.455312] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501240-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601240-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701240-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 778 [2024-06-06 10:29:44.555419] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 779 [2024-06-06 10:29:44.555466] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 780 [2024-06-06 10:29:44.606342] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501241-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 781 [2024-06-06 10:29:44.606378] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501241-0-0] [lt=32] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 782 [2024-06-06 10:29:44.625211] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601241-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 783 [2024-06-06 10:29:44.625257] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601241-0-0] [lt=44] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 784 [2024-06-06 10:29:44.626997] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701241-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 785 [2024-06-06 10:29:44.627026] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701241-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 786 [2024-06-06 10:29:44.655569] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 787 [2024-06-06 10:29:44.655599] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=29][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501241-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601241-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701241-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 788 [2024-06-06 10:29:44.655636] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 789 [2024-06-06 10:29:44.655643] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501241-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601241-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701241-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 790 [2024-06-06 10:29:44.729377] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 791 [2024-06-06 10:29:44.730031] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 792 [2024-06-06 10:29:44.730194] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 793 [2024-06-06 10:29:44.730360] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=19] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 794 [2024-06-06 10:29:44.755791] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 795 [2024-06-06 10:29:44.755893] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=62] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 796 [2024-06-06 10:29:44.781821] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501242-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 797 [2024-06-06 10:29:44.781880] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501242-0-0] [lt=52] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 798 [2024-06-06 10:29:44.800796] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601242-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 799 [2024-06-06 10:29:44.800837] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601242-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 800 [2024-06-06 10:29:44.805016] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701242-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 801 [2024-06-06 10:29:44.805051] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701242-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 802 [2024-06-06 10:29:44.829102] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=24] PNIO [ratelimit] time: 1717640984829099, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007615, add_bytes: 0 803 [2024-06-06 10:29:44.845999] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=20] PNIO [ratelimit] time: 1717640984845998, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007615, add_bytes: 0 804 [2024-06-06 10:29:44.856008] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=40][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 805 [2024-06-06 10:29:44.856036] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501242-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601242-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701242-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 806 [2024-06-06 10:29:44.856090] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=53][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 807 [2024-06-06 10:29:44.856099] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=7][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501242-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601242-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701242-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 808 [2024-06-06 10:29:44.956210] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 809 [2024-06-06 10:29:44.956284] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=49] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 810 [2024-06-06 10:29:44.968781] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=21] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 811 [2024-06-06 10:29:44.968816] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=17] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 812 [2024-06-06 10:29:44.969353] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 813 [2024-06-06 10:29:44.970344] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 814 [2024-06-06 10:29:44.993085] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501243-0-0] [lt=30] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 815 [2024-06-06 10:29:44.993135] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501243-0-0] [lt=46] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 816 [2024-06-06 10:29:45.008888] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=3/0, request done=3506/0, request doing=0/0) 817 [2024-06-06 10:29:45.009122] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=3/0, request done=3506/0, request doing=0/0) 818 [2024-06-06 10:29:45.009233] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=3/0, request done=3506/0, request doing=0/0) 819 [2024-06-06 10:29:45.009312] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=3/0, request done=3505/0, request doing=1/0) 820 [2024-06-06 10:29:45.009345] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 821 [2024-06-06 10:29:45.010120] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 822 [2024-06-06 10:29:45.010344] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=14] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 823 [2024-06-06 10:29:45.010423] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 824 [2024-06-06 10:29:45.014691] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601243-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 825 [2024-06-06 10:29:45.014719] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601243-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 826 [2024-06-06 10:29:45.016518] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701243-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 827 [2024-06-06 10:29:45.016551] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701243-0-0] [lt=31] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 828 [2024-06-06 10:29:45.056369] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=33][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 829 [2024-06-06 10:29:45.056400] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501243-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601243-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701243-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 830 [2024-06-06 10:29:45.056437] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 831 [2024-06-06 10:29:45.056443] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501243-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601243-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701243-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 832 [2024-06-06 10:29:45.156546] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 833 [2024-06-06 10:29:45.156596] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 834 [2024-06-06 10:29:45.190602] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501244-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 835 [2024-06-06 10:29:45.190642] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501244-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 836 [2024-06-06 10:29:45.209059] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601244-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 837 [2024-06-06 10:29:45.209091] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601244-0-0] [lt=29] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 838 [2024-06-06 10:29:45.210860] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701244-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 839 [2024-06-06 10:29:45.210901] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701244-0-0] [lt=39] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 840 [2024-06-06 10:29:45.256697] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 841 [2024-06-06 10:29:45.256724] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501244-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601244-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701244-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 842 [2024-06-06 10:29:45.256762] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=38][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 843 [2024-06-06 10:29:45.256769] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501244-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601244-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701244-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 844 [2024-06-06 10:29:45.356864] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 845 [2024-06-06 10:29:45.356918] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 846 [2024-06-06 10:29:45.385212] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501245-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 847 [2024-06-06 10:29:45.385254] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501245-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 848 [2024-06-06 10:29:45.404980] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601245-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 849 [2024-06-06 10:29:45.405020] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601245-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 850 [2024-06-06 10:29:45.406805] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701245-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 851 [2024-06-06 10:29:45.406844] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701245-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 852 [2024-06-06 10:29:45.457015] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 853 [2024-06-06 10:29:45.457041] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501245-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601245-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701245-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 854 [2024-06-06 10:29:45.457081] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=39][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 855 [2024-06-06 10:29:45.457088] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501245-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601245-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701245-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 856 [2024-06-06 10:29:45.557184] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 857 [2024-06-06 10:29:45.557237] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=33] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 858 [2024-06-06 10:29:45.583576] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501246-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 859 [2024-06-06 10:29:45.583621] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501246-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 860 [2024-06-06 10:29:45.602593] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601246-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 861 [2024-06-06 10:29:45.602630] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601246-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 862 [2024-06-06 10:29:45.604433] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701246-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 863 [2024-06-06 10:29:45.604468] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701246-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 864 [2024-06-06 10:29:45.657345] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=32][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 865 [2024-06-06 10:29:45.657370] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501246-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601246-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701246-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 866 [2024-06-06 10:29:45.657407] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 867 [2024-06-06 10:29:45.657414] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501246-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601246-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701246-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 868 [2024-06-06 10:29:45.729244] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=36] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 869 [2024-06-06 10:29:45.729305] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 870 [2024-06-06 10:29:45.730133] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=21] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 871 [2024-06-06 10:29:45.730347] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=29] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 872 [2024-06-06 10:29:45.757510] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 873 [2024-06-06 10:29:45.757571] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=41] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 874 [2024-06-06 10:29:45.790227] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501247-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 875 [2024-06-06 10:29:45.790266] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501247-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 876 [2024-06-06 10:29:45.809242] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601247-0-0] [lt=19] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 877 [2024-06-06 10:29:45.809271] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601247-0-0] [lt=28] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 878 [2024-06-06 10:29:45.811509] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701247-0-0] [lt=21] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 879 [2024-06-06 10:29:45.811549] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701247-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 880 [2024-06-06 10:29:45.836715] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1717640985836713, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007614, add_bytes: 0 881 [2024-06-06 10:29:45.853620] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=20] PNIO [ratelimit] time: 1717640985853618, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007620, add_bytes: 0 882 [2024-06-06 10:29:45.857634] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 883 [2024-06-06 10:29:45.857661] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501247-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601247-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701247-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 884 [2024-06-06 10:29:45.857701] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=39][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 885 [2024-06-06 10:29:45.857708] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501247-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601247-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701247-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 886 [2024-06-06 10:29:45.957804] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=23] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 887 [2024-06-06 10:29:45.957860] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=37] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 888 [2024-06-06 10:29:45.969327] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=22] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 889 [2024-06-06 10:29:45.969672] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 890 [2024-06-06 10:29:45.969672] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 891 [2024-06-06 10:29:45.969843] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=15] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 892 [2024-06-06 10:29:45.985140] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501248-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 893 [2024-06-06 10:29:45.985182] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501248-0-0] [lt=38] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 894 [2024-06-06 10:29:46.003753] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601248-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 895 [2024-06-06 10:29:46.003799] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601248-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 896 [2024-06-06 10:29:46.005809] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701248-0-0] [lt=19] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 897 [2024-06-06 10:29:46.005845] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701248-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 898 [2024-06-06 10:29:46.008712] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=3/0, request done=3510/0, request doing=0/0) 899 [2024-06-06 10:29:46.008800] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=3/0, request done=3510/0, request doing=0/0) 900 [2024-06-06 10:29:46.008866] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=3/0, request done=3510/0, request doing=0/0) 901 [2024-06-06 10:29:46.009188] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=15] [RPC EASY STAT](log_str=conn count=3/0, request done=3510/0, request doing=0/0) 902 [2024-06-06 10:29:46.009653] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 903 [2024-06-06 10:29:46.010349] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=15] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 904 [2024-06-06 10:29:46.010366] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 905 [2024-06-06 10:29:46.011448] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 906 [2024-06-06 10:29:46.057963] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 907 [2024-06-06 10:29:46.057994] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=30][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501248-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601248-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701248-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 908 [2024-06-06 10:29:46.058036] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=41][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 909 [2024-06-06 10:29:46.058042] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501248-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601248-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701248-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 910 [2024-06-06 10:29:46.117461] INFO [TLOG.PARSER] stat_ignored_tps (ob_log_cluster_id_filter.cpp:130) [20201][][T0][Y0-0000000000000000-0-0] [lt=4] [TPS_STAT] CLUSTER_ID_IGNORED_PART_TPS=0.000 911 [2024-06-06 10:29:46.158139] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 912 [2024-06-06 10:29:46.158188] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 913 [2024-06-06 10:29:46.190271] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-0000000000501249-0-0] [lt=17] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 914 [2024-06-06 10:29:46.190323] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-0000000000501249-0-0] [lt=48] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 915 [2024-06-06 10:29:46.209267] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-0000000000601249-0-0] [lt=24] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 916 [2024-06-06 10:29:46.209312] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-0000000000601249-0-0] [lt=42] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 917 [2024-06-06 10:29:46.211158] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-0000000000701249-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 918 [2024-06-06 10:29:46.211194] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-0000000000701249-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 919 [2024-06-06 10:29:46.258288] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 920 [2024-06-06 10:29:46.258318] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=29][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501249-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601249-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701249-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 921 [2024-06-06 10:29:46.258369] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=49][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 922 [2024-06-06 10:29:46.258375] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000501249-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000601249-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-0000000000701249-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 923 [2024-06-06 10:29:46.358473] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=23] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 924 [2024-06-06 10:29:46.358520] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 925 [2024-06-06 10:29:46.398924] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050124A-0-0] [lt=25] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 926 [2024-06-06 10:29:46.398976] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050124A-0-0] [lt=48] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 927 [2024-06-06 10:29:46.418128] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060124A-0-0] [lt=24] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 928 [2024-06-06 10:29:46.418168] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060124A-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 929 [2024-06-06 10:29:46.424106] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070124A-0-0] [lt=14] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 930 [2024-06-06 10:29:46.424153] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070124A-0-0] [lt=44] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 931 [2024-06-06 10:29:46.458618] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 932 [2024-06-06 10:29:46.458646] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=27][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050124A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060124A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070124A-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 933 [2024-06-06 10:29:46.458688] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=42][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 934 [2024-06-06 10:29:46.458695] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=6][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050124A-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060124A-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070124A-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 935 [2024-06-06 10:29:46.558793] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=21] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 936 [2024-06-06 10:29:46.558843] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 937 [2024-06-06 10:29:46.603954] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050124B-0-0] [lt=23] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 938 [2024-06-06 10:29:46.603993] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050124B-0-0] [lt=36] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 939 [2024-06-06 10:29:46.622748] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060124B-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 940 [2024-06-06 10:29:46.622788] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060124B-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 941 [2024-06-06 10:29:46.624701] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070124B-0-0] [lt=21] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 942 [2024-06-06 10:29:46.624726] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070124B-0-0] [lt=23] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 943 [2024-06-06 10:29:46.658938] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 944 [2024-06-06 10:29:46.658964] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050124B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060124B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070124B-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 945 [2024-06-06 10:29:46.659007] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=42][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 946 [2024-06-06 10:29:46.659022] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=14][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050124B-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060124B-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070124B-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 947 [2024-06-06 10:29:46.729356] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20157][RpcIO][T0][Y0-0000000000000000-0-0] [lt=18] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 948 [2024-06-06 10:29:46.729361] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20158][RpcIO][T0][Y0-0000000000000000-0-0] [lt=25] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 949 [2024-06-06 10:29:46.729686] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20156][RpcIO][T0][Y0-0000000000000000-0-0] [lt=12] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 950 [2024-06-06 10:29:46.729851] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20155][RpcIO][T0][Y0-0000000000000000-0-0] [lt=17] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 951 [2024-06-06 10:29:46.759130] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=33] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 952 [2024-06-06 10:29:46.759180] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=29] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 953 [2024-06-06 10:29:46.789082] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050124C-0-0] [lt=19] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 954 [2024-06-06 10:29:46.789120] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050124C-0-0] [lt=35] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 955 [2024-06-06 10:29:46.807709] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060124C-0-0] [lt=15] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 956 [2024-06-06 10:29:46.807752] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060124C-0-0] [lt=40] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 957 [2024-06-06 10:29:46.809739] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070124C-0-0] [lt=11] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 958 [2024-06-06 10:29:46.809766] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070124C-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 959 [2024-06-06 10:29:46.844339] INFO eloop_run (eloop.c:127) [20138][pnio1][T0][Y0-0000000000000000-0-0] [lt=21] PNIO [ratelimit] time: 1717640986844336, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007623, add_bytes: 0 960 [2024-06-06 10:29:46.859277] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 961 [2024-06-06 10:29:46.859300] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050124C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060124C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070124C-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 962 [2024-06-06 10:29:46.859349] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=47][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 963 [2024-06-06 10:29:46.859355] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050124C-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060124C-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070124C-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 964 [2024-06-06 10:29:46.861232] INFO eloop_run (eloop.c:127) [20142][pnio2][T0][Y0-0000000000000000-0-0] [lt=25] PNIO [ratelimit] time: 1717640986861230, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007612, add_bytes: 0 965 [2024-06-06 10:29:46.959453] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=22] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 966 [2024-06-06 10:29:46.959512] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=41] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 967 [2024-06-06 10:29:46.968890] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20162][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 968 [2024-06-06 10:29:46.969347] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20164][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=24] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 969 [2024-06-06 10:29:46.969349] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20160][BatchIO][T0][Y0-0000000000000000-0-0] [lt=13] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 970 [2024-06-06 10:29:46.970348] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20166][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 971 [2024-06-06 10:29:46.991364] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050124D-0-0] [lt=20] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 972 [2024-06-06 10:29:46.991404] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050124D-0-0] [lt=37] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 973 [2024-06-06 10:29:47.008411] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20170][RpcIO][T0][Y0-0000000000000000-0-0] [lt=16] [RPC EASY STAT](log_str=conn count=3/0, request done=3514/0, request doing=0/0) 974 [2024-06-06 10:29:47.008587] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20171][RpcIO][T0][Y0-0000000000000000-0-0] [lt=11] [RPC EASY STAT](log_str=conn count=3/0, request done=3513/0, request doing=1/0) 975 [2024-06-06 10:29:47.008711] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20172][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=3513/0, request doing=0/0) 976 [2024-06-06 10:29:47.008789] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20177][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=18] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 977 [2024-06-06 10:29:47.008858] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20173][RpcIO][T0][Y0-0000000000000000-0-0] [lt=14] [RPC EASY STAT](log_str=conn count=3/0, request done=3513/0, request doing=0/0) 978 [2024-06-06 10:29:47.010347] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [20175][BatchIO][T0][Y0-0000000000000000-0-0] [lt=16] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 979 [2024-06-06 10:29:47.010349] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [20181][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=13] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 980 [2024-06-06 10:29:47.010349] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [20179][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) 981 [2024-06-06 10:29:47.012759] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060124D-0-0] [lt=16] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 982 [2024-06-06 10:29:47.012795] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060124D-0-0] [lt=33] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 983 [2024-06-06 10:29:47.015990] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070124D-0-0] [lt=12] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 984 [2024-06-06 10:29:47.016019] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070124D-0-0] [lt=27] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 985 [2024-06-06 10:29:47.059614] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 986 [2024-06-06 10:29:47.059643] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=28][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050124D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060124D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070124D-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 987 [2024-06-06 10:29:47.059680] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=36][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 988 [2024-06-06 10:29:47.059686] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050124D-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060124D-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070124D-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 989 [2024-06-06 10:29:47.159787] INFO [TLOG] get_server_array_for_locate_start_lsn (ob_log_route_service.cpp:592) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=26] LSSvrList get_server_array_for_locate_start_lsn succ(ret=0, ret="OB_SUCCESS", router_key={cluster_id:126, tenant_id:1020, ls_id:{id:1}}) 990 [2024-06-06 10:29:47.159836] INFO [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:883) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=31] start lsn locate request launched(tls_id={tenant_id:1020, ls_id:{id:1}}, is_start_tstamp=true, start_tstamp="[2024-06-05 16:48:44.520923]", svr_cnt=3, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.50.54.157:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}, {svr:"10.88.8.206:2882", rpc_executed:false, rpc_err:0, svr_err:0, ls_err:0, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y0-0000000000000000-0-0}]) 991 [2024-06-06 10:29:47.187582] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20186][][T0][Y1BBE0A5818EC-000000000050124E-0-0] [lt=19] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.52.18.156:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 992 [2024-06-06 10:29:47.187619] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20186][][T0][Y1BBE0A5818EC-000000000050124E-0-0] [lt=34] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.52.18.156:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=1, svr_cnt=3) 993 [2024-06-06 10:29:47.206970] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20184][][T0][Y1BBE0A5818EC-000000000060124E-0-0] [lt=21] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.50.54.157:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 994 [2024-06-06 10:29:47.207015] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20184][][T0][Y1BBE0A5818EC-000000000060124E-0-0] [lt=41] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.50.54.157:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=2, svr_cnt=3) 995 [2024-06-06 10:29:47.208781] INFO [TLOG] req_start_lsn_by_tstamp (ob_log_rpc.cpp:107) [20183][][T0][Y1BBE0A5818EC-000000000070124E-0-0] [lt=13] rpc: request start LSN by tstamp(ret=0, ret="OB_SUCCESS", tenant_id=1020, svr="10.88.8.206:2882", timeout=60000000, req={rpc_ver:1, param_count:1, params:[{ls_id:{id:1}, start_ts_ns:1717577324520923023}], client_id:{client_addr:"10.88.24.236:7102", client_pid:7102}, flag:0}, resp={rpc_ver:1, err:0, result_count:1, result:[{err:-4233, start_lsn:{lsn:18446744073709551615}, start_ts_ns:-1}]}) 996 [2024-06-06 10:29:47.208809] INFO [TLOG.FETCHER] set_result (ob_log_start_lsn_locator.cpp:895) [20183][][T0][Y1BBE0A5818EC-000000000070124E-0-0] [lt=26] start lsn locate request of one server is done(ret=0, ret="OB_SUCCESS", tls_id={tenant_id:1020, ls_id:{id:1}}, svr_identity="10.88.8.206:2882", start_lsn={lsn:18446744073709551615}, start_log_tstamp=-1, start_tstamp_ns=1717577324520923023, delta=-1717577324520923024, rpc_err="OB_SUCCESS", svr_err="OB_SUCCESS", ls_err="OB_ERR_OUT_OF_LOWER_BOUND", cur_max_start_lsn_={lsn:18446744073709551615}, cur_max_start_log_tstamp_=-1, result_svr_list_idx_=-1, succ_locate_count_=0, svr_list_consumed=3, svr_cnt=3) 997 [2024-06-06 10:29:47.259935] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=25][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=-4016, file="ob_log_start_lsn_locator.cpp", line_no=929, info="request start lsn from all server fail") 998 [2024-06-06 10:29:47.259960] EDIAG [TLOG.FETCHER] get_result (ob_log_start_lsn_locator.cpp:929) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=24][errcode=-4016] request start lsn from all server fail(tls_id={tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns=1717577324520923023, svr_cnt=3, svr_list_consumed=3, result_svr_list_idx=-1, svr_list=[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050124E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060124E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070124E-0-0}]) BACKTRACE:0x16baee5d 0xb2df7b4 0xb2df261 0xb2deeca 0xb2decbb 0xb5c2e90 0xb3f77c0 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d 999 [2024-06-06 10:29:47.259995] ERROR issue_dba_error (ob_log.cpp:1875) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=34][errcode=-4388] Unexpected internal error happen, please checkout the internal errcode(errcode=0, file="ob_log_ls_fetch_ctx.cpp", line_no=908, info="start lsn locate fail") 1000 [2024-06-06 10:29:47.260001] EDIAG [TLOG.FETCHER] locate_lsn_ (ob_log_ls_fetch_ctx.cpp:908) [20187][][T0][Y1BBE0A5818EC-0000000000000001-0-0] [lt=5][errcode=0] start lsn locate fail(start_lsn_locate_req={state:2, tls_id:{tenant_id:1020, ls_id:{id:1}}, start_tstamp_ns:1717577324520923023, svr_cnt:3, svr_list_consumed:3, result_svr_list_idx:-1, cur_max_start_lsn:{lsn:18446744073709551615}, cur_max_start_log_tstamp:-1, succ_locate_count:0, svr_list:[{svr:"10.52.18.156:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000050124E-0-0}, {svr:"10.50.54.157:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000060124E-0-0}, {svr:"10.88.8.206:2882", rpc_executed:true, rpc_err:0, svr_err:0, ls_err:-4233, start_lsn:{lsn:18446744073709551615}, start_log_tstamp:18446744073709551615, trace_id:Y1BBE0A5818EC-000000000070124E-0-0}], fetching_mode:"Integrated Fetching Mode", archive_locate_rs:{start_lsn:{lsn:18446744073709551615}, loc_err:0}}) BACKTRACE:0x16baee5d 0xb3069c3 0x9157ca4 0xb306702 0xb25f881 0xb3f926f 0xb3f77d5 0xb3830da 0xb3824c3 0xb381918 0xb447e9f 0x7fd9479f4ea5 0x7fd94700eb0d