/home/admin/oceanbase/bin/observer -r 10.7.206.4:2882:2881;10.7.206.5:2882:2881;10.7.206.6:2882:2881 -p 2881 -P 2882 -z zone2 -n obcluster1 -c 1 -d /data/oceanbase -i eth0 -l WARN -o __min_full_resource_pool_memory=2147483648,memory_limit=48G,system_memory=8G,datafile_size=800G,log_disk_size=150G,cpu_count=0,enable_syslog_wf=False,enable_syslog_recycle=True,max_syslog_file_count=4 observer (OceanBase_CE 4.0.0.0) REVISION: 103000022023011215-05bbad0279302d7274e1b5ab79323a2c915c1981 BUILD_BRANCH: HEAD BUILD_TIME: Jan 12 2023 15:28:27 BUILD_FLAGS: RelWithDebInfo BUILD_INFO: Copyright (c) 2011-2022 OceanBase Inc. name= (9 segments) header 0: address= 0x200040 header 1: address= 0x200238 header 2: address= 0x200000 header 3: address= 0xd19a210 header 4: address= 0xd19a210 header 5: address= 0xd6ea100 header 6: address= 0x1d3c0a0 header 7: address= (nil) header 8: address= 0x200254 name=linux-vdso.so.1 (4 segments) header 0: address=0x7ffecb3fa000 header 1: address=0x7ffecb3fa398 header 2: address=0x7ffecb3fa7e8 header 3: address=0x7ffecb3fa83c name=/lib64/librt.so.1 (9 segments) header 0: address=0x7f47641e0000 header 1: address=0x7f47641e3000 header 2: address=0x7f47641e7000 header 3: address=0x7f47641e9ba0 header 4: address=0x7f47641e9bb0 header 5: address=0x7f47641e0238 header 6: address=0x7f47641e70d8 header 7: address=0x7f47641e0000 header 8: address=0x7f47641e9ba0 name=/lib64/libdl.so.2 (9 segments) header 0: address=0x7f47641db000 header 1: address=0x7f47641dc000 header 2: address=0x7f47641dd000 header 3: address=0x7f47641deca8 header 4: address=0x7f47641decc8 header 5: address=0x7f47641db238 header 6: address=0x7f47641dd0a4 header 7: address=0x7f47641db000 header 8: address=0x7f47641deca8 name=/lib64/libm.so.6 (9 segments) header 0: address=0x7f4764058000 header 1: address=0x7f4764067000 header 2: address=0x7f4764106000 header 3: address=0x7f47641d9c40 header 4: address=0x7f47641d9c50 header 5: address=0x7f4764058238 header 6: address=0x7f47641cd4e8 header 7: address=0x7f4764058000 header 8: address=0x7f47641d9c40 name=/lib64/libmariadb.so.3 (9 segments) header 0: address=0x7f4764006000 header 1: address=0x7f476401c000 header 2: address=0x7f4764041000 header 3: address=0x7f476404cd70 header 4: address=0x7f4764053740 header 5: address=0x7f4764006238 header 6: address=0x7f47640446e8 header 7: address=0x7f4764006000 header 8: address=0x7f476404cd70 name=/lib64/libaio.so.1 (9 segments) header 0: address=0x7f4764001000 header 1: address=0x7f4764002000 header 2: address=0x7f4764003000 header 3: address=0x7f4764004d68 header 4: address=0x7f4764004d78 header 5: address=0x7f4764001238 header 6: address=0x7f4764003000 header 7: address=0x7f4764001000 header 8: address=0x7f4764004d68 name=/lib64/libpthread.so.0 (11 segments) header 0: address=0x7f4763fe0040 header 1: address=0x7f4763ff6510 header 2: address=0x7f4763fe0000 header 3: address=0x7f4763fe7000 header 4: address=0x7f4763ff6000 header 5: address=0x7f4763ffb8e8 header 6: address=0x7f4763ffbaa0 header 7: address=0x7f4763fe02a8 header 8: address=0x7f4763ff6530 header 9: address=0x7f4763fe0000 header 10: address=0x7f4763ffb8e8 name=/lib64/libc.so.6 (12 segments) header 0: address=0x7f4763e26040 header 1: address=0x7f4763faced0 header 2: address=0x7f4763e26000 header 3: address=0x7f4763e4b000 header 4: address=0x7f4763f8f000 header 5: address=0x7f4763fd42e0 header 6: address=0x7f4763fd69a0 header 7: address=0x7f4763e262e0 header 8: address=0x7f4763fd42e0 header 9: address=0x7f4763faceec header 10: address=0x7f4763e26000 header 11: address=0x7f4763fd42e0 name=/lib64/ld-linux-x86-64.so.2 (9 segments) header 0: address=0x7f47641fa000 header 1: address=0x7f47641fb000 header 2: address=0x7f476421b000 header 3: address=0x7f47642225c0 header 4: address=0x7f4764222df8 header 5: address=0x7f47641fa238 header 6: address=0x7f476421e6fc header 7: address=0x7f47641fa000 header 8: address=0x7f47642225c0 name=/lib64/libz.so.1 (9 segments) header 0: address=0x7f4763e0c000 header 1: address=0x7f4763e0f000 header 2: address=0x7f4763e1d000 header 3: address=0x7f4763e24ad0 header 4: address=0x7f4763e24c30 header 5: address=0x7f4763e0c238 header 6: address=0x7f4763e21310 header 7: address=0x7f4763e0c000 header 8: address=0x7f4763e24ad0 name=/lib64/libssl.so.1.1 (9 segments) header 0: address=0x7f4763d79000 header 1: address=0x7f4763d96000 header 2: address=0x7f4763de3000 header 3: address=0x7f4763dfe310 header 4: address=0x7f4763e05f20 header 5: address=0x7f4763d79238 header 6: address=0x7f4763dedde0 header 7: address=0x7f4763d79000 header 8: address=0x7f4763dfe310 name=/lib64/libcrypto.so.1.1 (9 segments) header 0: address=0x7f4763a97000 header 1: address=0x7f4763b11000 header 2: address=0x7f4763cb1000 header 3: address=0x7f4763d45e88 header 4: address=0x7f4763d70848 header 5: address=0x7f4763a97238 header 6: address=0x7f4763cf6ca8 header 7: address=0x7f4763a97000 header 8: address=0x7f4763d45e88 [2023-03-27 16:47:31.847780] WARN [STORAGE.TRANS] getClock (ob_clock_generator.h:66) [2830644][][T0][Y0-0000000000000000-0-0] [lt=0] clock generator not inited [2023-03-27 16:47:31.848285] WARN [SERVER.OMT] get_tenant_config_with_lock (ob_tenant_config_mgr.cpp:355) [2830644][][T0][Y0-0000000000000000-0-0] [lt=87] failed to get fallback tenant config(fallback_tenant_id=1, ret=-4201) [2023-03-27 16:47:31.852713] WARN [STORAGE.TRANS] getClock (ob_clock_generator.h:66) [2830644][][T0][Y0-0000000000000000-0-0] [lt=10] clock generator not inited [2023-03-27 16:47:31.853153] WARN [SERVER.OMT] get_tenant_config_with_lock (ob_tenant_config_mgr.cpp:355) [2830644][][T0][Y0-0000000000000000-0-0] [lt=24] failed to get fallback tenant config(fallback_tenant_id=1, ret=-4201) [2023-03-27 16:47:31.853334] WARN [SHARE.LOCATION] get_leader_with_retry_until_timeout (ob_location_service.cpp:92) [2830644][][T0][Y0-0000000000000000-0-0] [lt=18] not init(ret=-4006, ret="OB_NOT_INIT") [2023-03-27 16:47:31.853355] WARN [SERVER] nonblock_get_leader (ob_inner_sql_connection.cpp:2118) [2830644][][T0][Y0-0000000000000000-0-0] [lt=18] get leader with retry until timeout failed(ret=-4006, ret="OB_NOT_INIT", tenant_id=1, ls_id={id:1}, leader="0.0.0.0:0", cluster_id=1, abs_timeout_us=1679906881853225, retry_interval_us=200000) [2023-03-27 16:47:31.853375] WARN [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:2030) [2830644][][T0][Y0-0000000000000000-0-0] [lt=18] nonblock get leader failed(ret=-4006, tenant_id=1, ls_id={id:1}, cluster_id=1) [2023-03-27 16:47:31.853382] WARN [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1164) [2830644][][T0][Y0-0000000000000000-0-0] [lt=7] retry_while_no_tenant_resource failed(ret=-4006, tenant_id=1) [2023-03-27 16:47:31.853388] WARN [SERVER] execute_read (ob_inner_sql_connection.cpp:1943) [2830644][][T0][Y0-0000000000000000-0-0] [lt=5] execute_read failed(ret=-4006, cluster_id=1, tenant_id=1) [2023-03-27 16:47:31.853396] WARN [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:108) [2830644][][T0][Y0-0000000000000000-0-0] [lt=8] query failed(ret=-4006, conn=0x7f4746d96050, start=1679906851853221, sql=select mode, size, latency, iops from __all_disk_io_calibration where svr_ip = "10.7.206.5" and svr_port = 2882 and storage_name = "DATA") [2023-03-27 16:47:31.853409] WARN [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:63) [2830644][][T0][Y0-0000000000000000-0-0] [lt=14] read failed(ret=-4006) [2023-03-27 16:47:31.853414] WARN [COMMON] parse_calibration_table (ob_io_calibration.cpp:823) [2830644][][T0][Y0-0000000000000000-0-0] [lt=4] query failed(ret=-4006, sql_string=select mode, size, latency, iops from __all_disk_io_calibration where svr_ip = "10.7.206.5" and svr_port = 2882 and storage_name = "DATA") [2023-03-27 16:47:31.853515] WARN [COMMON] read_from_table (ob_io_calibration.cpp:689) [2830644][][T0][Y0-0000000000000000-0-0] [lt=12] parse calibration data failed(ret=-4006) [2023-03-27 16:47:31.857634] WARN crc64_sse42_dispatch (ob_crc64.cpp:1139) [2830644][][T0][Y0-0000000000000000-0-0] [lt=12] Use CPU crc32 instructs for crc64 calculate [2023-03-27 16:47:31.884298] WARN create_tg_tenant (thread_mgr.h:976) [2830644][][T0][Y0-0000000000000000-0-0] [lt=19] create tg tenant but tenant tg helper is null(tg_def_id=13, tg_id=284) [2023-03-27 16:47:31.884790] WARN create_tg_tenant (thread_mgr.h:976) [2830644][][T0][Y0-0000000000000000-0-0] [lt=36] create tg tenant but tenant tg helper is null(tg_def_id=13, tg_id=285) [2023-03-27 16:47:31.885221] WARN create_tg_tenant (thread_mgr.h:976) [2830644][][T0][Y0-0000000000000000-0-0] [lt=13] create tg tenant but tenant tg helper is null(tg_def_id=13, tg_id=286) [2023-03-27 16:47:31.885640] WARN create_tg_tenant (thread_mgr.h:976) [2830644][][T0][Y0-0000000000000000-0-0] [lt=12] create tg tenant but tenant tg helper is null(tg_def_id=13, tg_id=287) [2023-03-27 16:47:31.886437] WARN create_tg_tenant (thread_mgr.h:976) [2830644][][T0][Y0-0000000000000000-0-0] [lt=14] create tg tenant but tenant tg helper is null(tg_def_id=13, tg_id=288) [2023-03-27 16:47:31.886908] WARN create_tg_tenant (thread_mgr.h:976) [2830644][][T0][Y0-0000000000000000-0-0] [lt=20] create tg tenant but tenant tg helper is null(tg_def_id=13, tg_id=289) [2023-03-27 16:47:31.887357] WARN create_tg_tenant (thread_mgr.h:976) [2830644][][T0][Y0-0000000000000000-0-0] [lt=13] create tg tenant but tenant tg helper is null(tg_def_id=13, tg_id=290) [2023-03-27 16:47:31.887761] WARN create_tg_tenant (thread_mgr.h:976) [2830644][][T0][Y0-0000000000000000-0-0] [lt=14] create tg tenant but tenant tg helper is null(tg_def_id=13, tg_id=291) [2023-03-27 16:47:31.888765] WARN [SERVER.OMT] init_cgroup_root_dir_ (ob_cgroup_ctrl.cpp:563) [2830644][][T0][Y0-0000000000000000-0-0] [lt=14] no cgroup directory found. disable cgroup support(cgroup_path="cgroup", ret=-4027) [2023-03-27 16:47:31.888788] WARN [SERVER.OMT] init (ob_cgroup_ctrl.cpp:46) [2830644][][T0][Y0-0000000000000000-0-0] [lt=26] init cgroup dir failed(ret=-4027, root_cgroup_="cgroup") [2023-03-27 16:47:31.888806] WARN [COMMON] get_instance (memory_dump.cpp:94) [2830644][][T0][Y0-0000000000000000-0-0] [lt=8] memory dump not init [2023-03-27 16:47:32.226997] WARN [COMMON] get_all_tenant_id (ob_tenant_mgr.cpp:168) [2830675][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=0] tenant manager not init(ret=-4006) [2023-03-27 16:47:32.227088] WARN [COMMON] get_all_tenant_id (ob_tenant_mem_limit_getter.cpp:64) [2830675][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=96] get all virtual tenant id failed(ret=-4006) [2023-03-27 16:47:32.227102] WARN [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:861) [2830675][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=8] Fail to get all tenant ids, (ret=-4006) [2023-03-27 16:47:32.318438] WARN [LIB] init (ob_tsc_timestamp.cpp:42) [2830644][][T0][Y0-0000000000000000-0-0] [lt=7] invariant TSC not support(ret=-4007) [2023-03-27 16:47:32.427627] WARN [COMMON] get_all_tenant_id (ob_tenant_mgr.cpp:168) [2830675][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=7] tenant manager not init(ret=-4006) [2023-03-27 16:47:32.427671] WARN [COMMON] get_all_tenant_id (ob_tenant_mem_limit_getter.cpp:64) [2830675][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=50] get all virtual tenant id failed(ret=-4006) [2023-03-27 16:47:32.427680] WARN [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:861) [2830675][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=8] Fail to get all tenant ids, (ret=-4006) [2023-03-27 16:47:32.483985] WARN [SERVER] get_network_speed_from_config_file (ob_server.cpp:2197) [2830644][][T0][Y0-0000000000000000-0-0] [lt=57] NIC Config file doesn't exist, auto detecting(nic_rate_path="etc/nic.rate.config", ret=-4027, ret="OB_FILE_NOT_EXIST") [2023-03-27 16:47:32.484168] WARN [SERVER] get_network_speed_from_sysfs (ob_server.cpp:2113) [2830644][][T0][Y0-0000000000000000-0-0] [lt=51] get invalid Ethernet speed, use default(devname="eth0") [2023-03-27 16:47:32.593604] ERROR [CLOG] allocate_block_at_tmp_dir_ (ob_server_log_block_mgr.cpp:905) [2830644][][T0][Y0-0000000000000000-0-0] [lt=14] ::fallocate failed(ret=-4009, this={dir::"/data/oceanbase/clog/log_pool", dir_fd:14, meta_fd:15, log_pool_meta:{curr_total_size:0, next_total_size:161061273600, status:1}, min_block_id:0, max_block_id:0, is_inited:true}, dir_fd=65, block_id=0, errno=95) BACKTRACE:0xb61bbbb 0xb60d4f6 0x3c10408 0x3c10102 0x3c0fefe 0x3c0fd48 0x4403be3 0x44033eb 0x44017ab 0x4400b32 0x43f75a7 0x43f6981 0x43f66c5 0x5be6ca0 0x3be6f34 0x7f4763e4bb47 0x3be5c9e [2023-03-27 16:47:32.593823] ERROR [CLOG] allocate_blocks_at_tmp_dir_ (ob_server_log_block_mgr.cpp:875) [2830644][][T0][Y0-0000000000000000-0-0] [lt=214] allocate_block_at_tmp_dir_ failed(ret=-4009, this={dir::"/data/oceanbase/clog/log_pool", dir_fd:14, meta_fd:15, log_pool_meta:{curr_total_size:0, next_total_size:161061273600, status:1}, min_block_id:0, max_block_id:0, is_inited:true}, dir_fd=65, block_id=0) BACKTRACE:0xb61bbbb 0xb60d4f6 0x3c13c18 0x3c13912 0x3c1370f 0x3c09f61 0x44035a4 0x44017e8 0x4400b32 0x43f75a7 0x43f6981 0x43f66c5 0x5be6ca0 0x3be6f34 0x7f4763e4bb47 0x3be5c9e [2023-03-27 16:47:32.593860] ERROR [CLOG] do_expand_ (ob_server_log_block_mgr.cpp:787) [2830644][][T0][Y0-0000000000000000-0-0] [lt=35] allocate_blocks_at_ failed(ret=-4009, this={dir::"/data/oceanbase/clog/log_pool", dir_fd:14, meta_fd:15, log_pool_meta:{curr_total_size:0, next_total_size:161061273600, status:1}, min_block_id:0, max_block_id:0, is_inited:true}) BACKTRACE:0xb61bbbb 0xb60d4f6 0x3c12f81 0x3c12c84 0x3c12a99 0x3bf0efb 0x4401919 0x4400c2f 0x43f75a7 0x43f6981 0x43f66c5 0x5be6ca0 0x3be6f34 0x7f4763e4bb47 0x3be5c9e [2023-03-27 16:47:32.596812] ERROR [CLOG] do_resize_ (ob_server_log_block_mgr.cpp:738) [2830644][][T0][Y0-0000000000000000-0-0] [lt=39] do_expand_ failed(ret=-4009, this={dir::"/data/oceanbase/clog/log_pool", dir_fd:14, meta_fd:15, log_pool_meta:{curr_total_size:0, next_total_size:161061273600, status:1}, min_block_id:0, max_block_id:0, is_inited:true}) BACKTRACE:0xb61bbbb 0xb60d4f6 0x3c12f81 0x3c12c84 0x3c12a99 0x3bf0efb 0x4400d3d 0x43f76af 0x43f6981 0x43f66c5 0x5be6ca0 0x3be6f34 0x7f4763e4bb47 0x3be5c9e [2023-03-27 16:47:32.597663] ERROR [CLOG] resize (ob_server_log_block_mgr.cpp:197) [2830644][][T0][Y0-0000000000000000-0-0] [lt=37] do_resize_ failed(ret=-4009, this={dir::"/data/oceanbase/clog/log_pool", dir_fd:14, meta_fd:15, log_pool_meta:{curr_total_size:0, next_total_size:0, status:0}, min_block_id:0, max_block_id:0, is_inited:true}, old_log_pool_meta={curr_total_size:0, next_total_size:0, status:0}, new_log_pool_meta={curr_total_size:0, next_total_size:161061273600, status:1}) BACKTRACE:0xb61bbbb 0xb60d4f6 0x3c13c18 0x3c13912 0x3c1370f 0x3c09f61 0x43f77e6 0x43f6a93 0x43f66c5 0x5be6ca0 0x3be6f34 0x7f4763e4bb47 0x3be5c9e [2023-03-27 16:47:32.597718] ERROR [CLOG] reserve (ob_server_log_block_mgr.cpp:155) [2830644][][T0][Y0-0000000000000000-0-0] [lt=55] resize failed(ret=-4009, this={dir::"/data/oceanbase/clog/log_pool", dir_fd:14, meta_fd:15, log_pool_meta:{curr_total_size:0, next_total_size:0, status:0}, min_block_id:0, max_block_id:0, is_inited:true}) BACKTRACE:0xb61bbbb 0xb60d4f6 0x3c12f81 0x3c12c84 0x3c12a99 0x3bf0efb 0x43f6bf1 0x43f6711 0x5be6ca0 0x3be6f34 0x7f4763e4bb47 0x3be5c9e [2023-03-27 16:47:32.597740] ERROR [SERVER] start (ob_server.cpp:578) [2830644][][T0][Y0-0000000000000000-0-0] [lt=17] fail to reserve log pool(ret=-4009, ret="OB_IO_ERROR") BACKTRACE:0xb61bbbb 0xb60d4f6 0x3c12f81 0x3c12c84 0x3c12a99 0x3bf0efb 0x5be830f 0x5be730c 0x3be6f34 0x7f4763e4bb47 0x3be5c9e [2023-03-27 16:47:32.597904] ERROR [SERVER] start (ob_server.cpp:793) [2830644][][T0][Y0-0000000000000000-0-0] [lt=164] failure occurs, try to set stop and wait(ret=-4009, ret="OB_IO_ERROR") BACKTRACE:0xb61bbbb 0xb60d4f6 0x3c12f81 0x3c12c84 0x3c12a99 0x3bf0efb 0x5beba69 0x5be783f 0x3be6f34 0x7f4763e4bb47 0x3be5c9e [2023-03-27 16:47:32.598514] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=0] connection pool stoped(ret=-8002) [2023-03-27 16:47:32.598568] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=60] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:32.598575] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=5] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:32.598582] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:32.598587] WARN [SHARE] process_task (ob_event_history_table_operator.cpp:300) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1679906852598218), 'root_service', 'stop_rootservice', 'self_addr', '"10.7.206.5:2882"', '10.7.206.5', 2882), ret=-8002) [2023-03-27 16:47:32.598606] WARN [SHARE] process (ob_event_history_table_operator.cpp:161) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=19] process_task failed(sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1679906852598218), 'root_service', 'stop_rootservice', 'self_addr', '"10.7.206.5:2882"', '10.7.206.5', 2882), is_delete=false, ret=-8002) [2023-03-27 16:47:32.598717] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=104] connection pool stoped(ret=-8002) [2023-03-27 16:47:32.598723] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=5] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:32.598726] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=3] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:32.598729] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=3] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:32.598733] WARN [SHARE] process_task (ob_event_history_table_operator.cpp:300) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=3] execute sql failed(sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1679906852598657), 'backup_lease', 'stop backup lease', 'local_addr', '"10.7.206.5:2882"', 'expect_round', 1, '10.7.206.5', 2882), ret=-8002) [2023-03-27 16:47:32.598736] WARN [SHARE] process (ob_event_history_table_operator.cpp:161) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=4] process_task failed(sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1679906852598657), 'backup_lease', 'stop backup lease', 'local_addr', '"10.7.206.5:2882"', 'expect_round', 1, '10.7.206.5', 2882), is_delete=false, ret=-8002) [2023-03-27 16:47:32.598917] WARN [SERVER] stop (ob_server.cpp:987) [2830644][][T0][Y0-0000000000000000-0-0] [lt=13] server log block mgr stopped [2023-03-27 16:47:32.598943] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=3] connection pool stoped(ret=-8002) [2023-03-27 16:47:32.598952] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=0] connection pool stoped(ret=-8002) [2023-03-27 16:47:32.598966] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=25] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:32.598967] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=16] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:32.598973] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=6] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:32.598974] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:32.598979] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=6] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:32.598981] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:32.598994] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:32.599016] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=19] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:32.598986] WARN [SHARE] process_task (ob_event_history_table_operator.cpp:300) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=6] execute sql failed(sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1679906852598869), 'root_service', 'finish_stop_thread', 'ret', 0, 'ret', 'OB_SUCCESS', 'self_addr', '"10.7.206.5:2882"', '10.7.206.5', 2882), ret=-8002) [2023-03-27 16:47:32.599027] WARN [SHARE] process (ob_event_history_table_operator.cpp:161) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=40] process_task failed(sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1679906852598869), 'root_service', 'finish_stop_thread', 'ret', 0, 'ret', 'OB_SUCCESS', 'self_addr', '"10.7.206.5:2882"', '10.7.206.5', 2882), is_delete=false, ret=-8002) [2023-03-27 16:47:32.599546] WARN [SERVER.OMT] get_tenant_config_with_lock (ob_tenant_config_mgr.cpp:355) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=0] failed to get fallback tenant config(fallback_tenant_id=1, ret=-4201) [2023-03-27 16:47:32.599788] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=73] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:32.669337] WARN [RPC.FRAME] high_prio_rpc_shutdown (ob_net_easy.cpp:859) [2830644][][T0][Y0-0000000000000000-0-0] [lt=14] easy net hasn't started(ret=-4016) [2023-03-27 16:47:32.669373] WARN [SERVER] stop (ob_server.cpp:1016) [2830644][][T0][Y0-0000000000000000-0-0] [lt=38] fail to shutdown high priority rpc(ret=-4016, ret="OB_ERR_UNEXPECTED") [2023-03-27 16:47:32.799964] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=30] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:33.000157] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=51] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:33.200337] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=76] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:33.275302] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=7] connection pool stoped(ret=-8002) [2023-03-27 16:47:33.275337] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=40] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:33.275345] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=6] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:33.275351] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:33.275356] WARN [SHARE] process_task (ob_event_history_table_operator.cpp:300) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1679906853275195), 'root_service', 'finish_wait_stop', 'cost', 8892, '10.7.206.5', 2882), ret=-8002) [2023-03-27 16:47:33.275364] WARN [SHARE] process (ob_event_history_table_operator.cpp:161) [2830731][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=8] process_task failed(sql=INSERT INTO __all_rootservice_event_history (gmt_create, module, event, name1, value1, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1679906853275195), 'root_service', 'finish_wait_stop', 'cost', 8892, '10.7.206.5', 2882), is_delete=false, ret=-8002) [2023-03-27 16:47:33.400512] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=54] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:33.599366] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] connection pool stoped(ret=-8002) [2023-03-27 16:47:33.599400] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=41] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:33.599410] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:33.599416] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:33.599422] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:33.599440] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:33.600715] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=55] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:33.800907] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=58] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:34.001098] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=54] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:34.201353] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=83] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:34.401565] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=66] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:34.599752] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] connection pool stoped(ret=-8002) [2023-03-27 16:47:34.599787] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=42] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:34.599798] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:34.599804] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:34.599809] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:34.599821] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:34.601781] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=92] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:34.801971] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=60] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:35.002225] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=58] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:35.202447] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=69] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:35.402705] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=89] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:35.600178] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=39] connection pool stoped(ret=-8002) [2023-03-27 16:47:35.600222] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=50] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:35.600233] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:35.600241] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=10] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:35.600248] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:35.600262] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=10] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:35.602914] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=87] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:35.803112] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=74] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:36.003409] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=98] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:36.203630] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=96] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:36.403855] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=74] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:36.600519] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] connection pool stoped(ret=-8002) [2023-03-27 16:47:36.600581] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=69] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:36.600591] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:36.600598] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:36.600603] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:36.600614] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:36.604067] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=83] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:36.804296] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=116] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:36.822837] WARN [SERVER.OMT] get_tenant_base_with_lock (ob_multi_tenant.cpp:150) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=0] get tenant from omt failed(ret=-5150, tenant_id=1) [2023-03-27 16:47:36.823257] WARN [SHARE] switch_to (ob_tenant_base.cpp:489) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=56] switch tenant fail(tenant_id=1, ret=-5150, lbt()="0xb61bbbb 0xa097f8c 0x3a0fa2d 0xa37fac5 0x45e50c5 0x4651b58 0x45e5248 0x3a391f1 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260") [2023-03-27 16:47:36.823290] WARN [SERVER.OMT] operator() (ob_tenant_timezone_mgr.cpp:72) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=32] fail to update time zone info(ret=-5150) [2023-03-27 16:47:36.823299] WARN foreach_refactored (ob_hashtable.h:1318) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=8] fail to do call back, ret=-5150 [2023-03-27 16:47:36.823305] WARN [SERVER.OMT] runTimerTask (ob_tenant_timezone_mgr.cpp:86) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=7] update tenant time zone failed(ret=-5150) [2023-03-27 16:47:37.004490] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=74] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:37.204703] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=73] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:37.404911] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=81] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:37.600844] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] connection pool stoped(ret=-8002) [2023-03-27 16:47:37.600872] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=33] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:37.600881] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:37.600888] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:37.600894] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:37.600905] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:37.605109] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=74] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:37.805320] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=94] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:38.005514] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=76] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:38.206279] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=73] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:38.406686] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=480] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:38.587596] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=9] too much time used to wait connection release, may be connection leak(used_time_ms=5000640, used connection count=1) [2023-03-27 16:47:38.587653] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=65] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:38.587683] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=7] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:38.601189] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] connection pool stoped(ret=-8002) [2023-03-27 16:47:38.601229] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=45] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:38.601238] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:38.601246] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:38.601250] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:38.601261] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:38.606868] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=72] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:38.807063] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=72] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:39.007362] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=117] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:39.207647] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=103] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:39.407911] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=80] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:39.587836] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=11] too much time used to wait connection release, may be connection leak(used_time_ms=6000882, used connection count=1) [2023-03-27 16:47:39.587899] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=68] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:39.587929] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=7] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:39.601604] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] connection pool stoped(ret=-8002) [2023-03-27 16:47:39.601652] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=55] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:39.601669] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=13] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:39.601720] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=14] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:39.601735] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=50] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:39.601752] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=13] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:39.608189] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=79] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:39.808410] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=82] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:40.008694] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=71] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:40.209287] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=86] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:40.409534] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=78] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:40.588068] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=11] too much time used to wait connection release, may be connection leak(used_time_ms=7001114, used connection count=1) [2023-03-27 16:47:40.588121] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=60] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:40.588194] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=7] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:40.602348] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] connection pool stoped(ret=-8002) [2023-03-27 16:47:40.602399] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=56] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:40.602410] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:40.602489] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=79] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:40.602497] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:40.602515] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=15] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:40.609892] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=74] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:40.810203] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=110] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:41.010474] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=80] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:41.210724] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=108] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:41.411076] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=68] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:41.588320] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=13] too much time used to wait connection release, may be connection leak(used_time_ms=8001366, used connection count=1) [2023-03-27 16:47:41.588370] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=56] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:41.588399] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=6] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:41.602867] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=14] connection pool stoped(ret=-8002) [2023-03-27 16:47:41.602939] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=79] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:41.602948] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:41.602954] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:41.602959] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:41.602971] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:41.611360] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=103] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:41.811638] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=80] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:41.823452] WARN [SERVER.OMT] get_tenant_base_with_lock (ob_multi_tenant.cpp:150) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=5] get tenant from omt failed(ret=-5150, tenant_id=1) [2023-03-27 16:47:41.823511] WARN [SHARE] switch_to (ob_tenant_base.cpp:489) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=47] switch tenant fail(tenant_id=1, ret=-5150, lbt()="0xb61bbbb 0xa097f8c 0x3a0fa2d 0xa37fac5 0x45e50c5 0x4651b58 0x45e5248 0x3a391f1 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260") [2023-03-27 16:47:41.823559] WARN [SERVER.OMT] operator() (ob_tenant_timezone_mgr.cpp:72) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=44] fail to update time zone info(ret=-5150) [2023-03-27 16:47:41.823566] WARN foreach_refactored (ob_hashtable.h:1318) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=6] fail to do call back, ret=-5150 [2023-03-27 16:47:41.823571] WARN [SERVER.OMT] runTimerTask (ob_tenant_timezone_mgr.cpp:86) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=5] update tenant time zone failed(ret=-5150) [2023-03-27 16:47:42.011920] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=92] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:42.212216] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=85] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:42.412464] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=77] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:42.588602] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=10] too much time used to wait connection release, may be connection leak(used_time_ms=9001648, used connection count=1) [2023-03-27 16:47:42.588662] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=65] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:42.588697] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=8] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:42.603312] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] connection pool stoped(ret=-8002) [2023-03-27 16:47:42.603355] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=48] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:42.603364] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:42.603372] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:42.603377] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:42.603389] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:42.612734] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=76] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:42.813184] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=79] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:43.013502] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=197] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:43.213784] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=80] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:43.414024] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=114] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:43.588877] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=12] too much time used to wait connection release, may be connection leak(used_time_ms=10001918, used connection count=1) [2023-03-27 16:47:43.588944] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=72] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:43.588994] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=7] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:43.603747] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] connection pool stoped(ret=-8002) [2023-03-27 16:47:43.603788] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=46] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:43.603805] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=14] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:43.603823] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=20] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:43.603839] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=14] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:43.603985] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=143] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:43.614240] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=79] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:43.814438] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=67] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:44.014649] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=80] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:44.214853] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=72] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:44.415104] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=77] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:44.589163] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=12] too much time used to wait connection release, may be connection leak(used_time_ms=11002183, used connection count=1) [2023-03-27 16:47:44.589223] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=90] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:44.589258] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=8] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:44.604259] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=15] connection pool stoped(ret=-8002) [2023-03-27 16:47:44.604293] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=40] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:44.604339] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=43] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:44.604345] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:44.604351] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:44.604367] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=13] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:44.615359] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=99] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:44.815615] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=75] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:45.015922] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=73] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:45.216283] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=115] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:45.416529] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=104] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:45.589386] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=12] too much time used to wait connection release, may be connection leak(used_time_ms=12002431, used connection count=1) [2023-03-27 16:47:45.589440] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=61] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:45.589471] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=7] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:45.604663] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] connection pool stoped(ret=-8002) [2023-03-27 16:47:45.604702] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=44] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:45.604712] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:45.604721] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=10] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:45.604731] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:45.604756] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=20] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:45.616792] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=102] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:45.817071] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=78] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:46.017365] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=125] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:46.217614] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=79] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:46.417915] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=76] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:46.589649] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=51] too much time used to wait connection release, may be connection leak(used_time_ms=13002695, used connection count=1) [2023-03-27 16:47:46.589752] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=109] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:46.589782] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=6] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:46.605248] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] connection pool stoped(ret=-8002) [2023-03-27 16:47:46.605288] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=44] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:46.605297] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:46.605306] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:46.605313] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:46.605328] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=12] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:46.618203] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=88] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:46.818489] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=98] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:46.823708] WARN [SERVER.OMT] get_tenant_base_with_lock (ob_multi_tenant.cpp:150) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=5] get tenant from omt failed(ret=-5150, tenant_id=1) [2023-03-27 16:47:46.823767] WARN [SHARE] switch_to (ob_tenant_base.cpp:489) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=47] switch tenant fail(tenant_id=1, ret=-5150, lbt()="0xb61bbbb 0xa097f8c 0x3a0fa2d 0xa37fac5 0x45e50c5 0x4651b58 0x45e5248 0x3a391f1 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260") [2023-03-27 16:47:46.823781] WARN [SERVER.OMT] operator() (ob_tenant_timezone_mgr.cpp:72) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=10] fail to update time zone info(ret=-5150) [2023-03-27 16:47:46.823789] WARN foreach_refactored (ob_hashtable.h:1318) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=6] fail to do call back, ret=-5150 [2023-03-27 16:47:46.823795] WARN [SERVER.OMT] runTimerTask (ob_tenant_timezone_mgr.cpp:86) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=7] update tenant time zone failed(ret=-5150) [2023-03-27 16:47:47.018712] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=87] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:47.219168] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=76] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:47.419772] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=116] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:47.589915] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=11] too much time used to wait connection release, may be connection leak(used_time_ms=14002964, used connection count=1) [2023-03-27 16:47:47.589960] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=49] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:47.589989] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=7] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:47.605815] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] connection pool stoped(ret=-8002) [2023-03-27 16:47:47.605867] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=58] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:47.605881] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=11] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:47.605925] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=45] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:47.605930] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:47.605941] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:47.620175] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=207] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:47.820454] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=109] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:48.020694] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=93] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:48.220912] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=75] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:48.421131] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=75] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:48.590217] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=10] too much time used to wait connection release, may be connection leak(used_time_ms=15003262, used connection count=1) [2023-03-27 16:47:48.590271] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=61] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:48.590301] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=7] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:48.606240] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] connection pool stoped(ret=-8002) [2023-03-27 16:47:48.606307] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=72] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:48.606317] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:48.606325] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:48.606331] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:48.606344] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=10] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:48.621365] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=97] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:48.821566] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=74] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:49.021799] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=78] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:49.222036] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=83] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:49.422267] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=91] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:49.590479] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=11] too much time used to wait connection release, may be connection leak(used_time_ms=16003525, used connection count=1) [2023-03-27 16:47:49.590538] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=65] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:49.590605] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=44] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:49.606616] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] connection pool stoped(ret=-8002) [2023-03-27 16:47:49.606649] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=38] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:49.606658] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:49.606665] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:49.606670] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:49.606681] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:49.622634] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=77] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:49.823328] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=207] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:50.023806] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=259] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:50.224108] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=82] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:50.424424] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=145] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:50.590734] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=10] too much time used to wait connection release, may be connection leak(used_time_ms=17003781, used connection count=1) [2023-03-27 16:47:50.590783] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=53] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:50.590811] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=6] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:50.607039] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] connection pool stoped(ret=-8002) [2023-03-27 16:47:50.607081] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=47] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:50.607091] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:50.607098] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:50.607104] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:50.607115] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=10] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:50.624660] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=76] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:50.824936] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=86] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:51.025214] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=87] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:51.225490] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=77] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:51.425801] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=92] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:51.590994] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=10] too much time used to wait connection release, may be connection leak(used_time_ms=18004036, used connection count=1) [2023-03-27 16:47:51.591063] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=77] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:51.591111] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=9] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:51.607502] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] connection pool stoped(ret=-8002) [2023-03-27 16:47:51.607538] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=42] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:51.607547] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:51.607554] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:51.607559] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:51.607570] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:51.626092] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=99] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:51.823932] WARN [SERVER.OMT] get_tenant_base_with_lock (ob_multi_tenant.cpp:150) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=5] get tenant from omt failed(ret=-5150, tenant_id=1) [2023-03-27 16:47:51.823999] WARN [SHARE] switch_to (ob_tenant_base.cpp:489) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=54] switch tenant fail(tenant_id=1, ret=-5150, lbt()="0xb61bbbb 0xa097f8c 0x3a0fa2d 0xa37fac5 0x45e50c5 0x4651b58 0x45e5248 0x3a391f1 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260") [2023-03-27 16:47:51.824013] WARN [SERVER.OMT] operator() (ob_tenant_timezone_mgr.cpp:72) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=12] fail to update time zone info(ret=-5150) [2023-03-27 16:47:51.824022] WARN foreach_refactored (ob_hashtable.h:1318) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=7] fail to do call back, ret=-5150 [2023-03-27 16:47:51.824027] WARN [SERVER.OMT] runTimerTask (ob_tenant_timezone_mgr.cpp:86) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=6] update tenant time zone failed(ret=-5150) [2023-03-27 16:47:51.826326] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=98] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:52.026573] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=64] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:52.226874] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=105] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:52.427121] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=87] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:52.591243] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=12] too much time used to wait connection release, may be connection leak(used_time_ms=19004284, used connection count=1) [2023-03-27 16:47:52.591305] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=72] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:52.591346] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=9] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:52.607929] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] connection pool stoped(ret=-8002) [2023-03-27 16:47:52.607990] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=61] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:52.608000] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:52.608011] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=12] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:52.608019] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:52.608034] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=11] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:52.627408] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=100] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:52.827680] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=86] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:53.027939] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=78] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:53.228210] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=79] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:53.428518] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=117] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:53.591577] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=116] too much time used to wait connection release, may be connection leak(used_time_ms=20004618, used connection count=1) [2023-03-27 16:47:53.591657] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=89] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:53.591697] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=8] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:53.608377] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=13] connection pool stoped(ret=-8002) [2023-03-27 16:47:53.608413] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=42] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:53.608423] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:53.608439] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=17] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:53.608444] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:53.608456] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:53.628779] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=77] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:53.829062] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=92] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:54.029363] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=113] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:54.229755] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=86] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:54.430102] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=157] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:54.591829] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=13] too much time used to wait connection release, may be connection leak(used_time_ms=21004872, used connection count=1) [2023-03-27 16:47:54.591899] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=76] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:54.591954] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=14] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:54.608848] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] connection pool stoped(ret=-8002) [2023-03-27 16:47:54.608901] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=59] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:54.608919] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=14] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:54.608928] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=10] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:54.608936] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:54.608949] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=10] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:54.630406] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=126] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:54.830642] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=76] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:55.030908] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=83] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:55.231165] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=91] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:55.431388] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=82] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:55.592166] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=26] too much time used to wait connection release, may be connection leak(used_time_ms=22005181, used connection count=1) [2023-03-27 16:47:55.592227] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=95] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:55.592271] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=8] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:55.609452] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=24] connection pool stoped(ret=-8002) [2023-03-27 16:47:55.609487] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=42] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:55.609497] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:55.609515] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=19] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:55.609521] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:55.609561] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=37] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:55.631631] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=83] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:55.831839] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=84] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:56.032097] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=79] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:56.232514] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=124] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:56.432716] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=72] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:56.592430] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=14] too much time used to wait connection release, may be connection leak(used_time_ms=23005471, used connection count=1) [2023-03-27 16:47:56.592497] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=75] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:56.592540] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=9] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:56.610100] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=14] connection pool stoped(ret=-8002) [2023-03-27 16:47:56.610168] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=75] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:56.610179] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:56.610217] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=38] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:56.610222] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:56.610234] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:56.632977] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=95] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:56.824207] WARN [SERVER.OMT] get_tenant_base_with_lock (ob_multi_tenant.cpp:150) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=5] get tenant from omt failed(ret=-5150, tenant_id=1) [2023-03-27 16:47:56.824285] WARN [SHARE] switch_to (ob_tenant_base.cpp:489) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=62] switch tenant fail(tenant_id=1, ret=-5150, lbt()="0xb61bbbb 0xa097f8c 0x3a0fa2d 0xa37fac5 0x45e50c5 0x4651b58 0x45e5248 0x3a391f1 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260") [2023-03-27 16:47:56.824307] WARN [SERVER.OMT] operator() (ob_tenant_timezone_mgr.cpp:72) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=11] fail to update time zone info(ret=-5150) [2023-03-27 16:47:56.824319] WARN foreach_refactored (ob_hashtable.h:1318) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=7] fail to do call back, ret=-5150 [2023-03-27 16:47:56.824382] WARN [SERVER.OMT] runTimerTask (ob_tenant_timezone_mgr.cpp:86) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=62] update tenant time zone failed(ret=-5150) [2023-03-27 16:47:56.833290] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=87] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:57.033570] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=81] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:57.233971] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=83] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:57.434242] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=97] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:57.592711] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=13] too much time used to wait connection release, may be connection leak(used_time_ms=24005752, used connection count=1) [2023-03-27 16:47:57.592776] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=75] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:57.592829] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=15] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:57.610542] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] connection pool stoped(ret=-8002) [2023-03-27 16:47:57.610583] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=47] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:57.610592] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:57.610598] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:57.610603] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=4] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:57.610615] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:57.634567] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=77] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:57.834802] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=82] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:58.035067] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=75] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:58.235375] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=126] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:58.435635] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=78] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:58.593019] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=14] too much time used to wait connection release, may be connection leak(used_time_ms=25006060, used connection count=1) [2023-03-27 16:47:58.593076] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=67] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:58.593115] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=8] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:58.610940] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] connection pool stoped(ret=-8002) [2023-03-27 16:47:58.610972] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=38] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:58.610980] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:58.610987] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=7] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:58.610992] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:58.611003] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:58.635893] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=80] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:58.836808] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=101] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:59.037191] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=149] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:59.237483] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=116] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:59.437731] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=80] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:59.593253] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=12] too much time used to wait connection release, may be connection leak(used_time_ms=26006294, used connection count=1) [2023-03-27 16:47:59.593314] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=70] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:47:59.593353] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=8] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:47:59.611371] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] connection pool stoped(ret=-8002) [2023-03-27 16:47:59.611419] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=52] alloc connection from pool failed(ret=-8002) [2023-03-27 16:47:59.611430] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=8] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:59.611438] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=10] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:47:59.611476] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=36] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:47:59.611492] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=12] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:47:59.637974] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=76] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:47:59.838218] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=88] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:00.038445] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=86] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:00.238659] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=78] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:00.438864] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=81] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:00.593491] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=11] too much time used to wait connection release, may be connection leak(used_time_ms=27006530, used connection count=1) [2023-03-27 16:48:00.593554] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=74] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:48:00.593598] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=7] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:48:00.611829] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] connection pool stoped(ret=-8002) [2023-03-27 16:48:00.611866] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=43] alloc connection from pool failed(ret=-8002) [2023-03-27 16:48:00.611907] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=39] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:48:00.611915] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:48:00.611921] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=5] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:48:00.611933] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=10] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:48:00.639083] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=91] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:00.839342] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=103] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:01.039613] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=112] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:01.239923] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=87] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:01.440211] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=116] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:01.593794] WARN [SERVER] wait (ob_inner_sql_connection_pool.cpp:415) [2830644][][T0][Y0-0000000000000000-0-0] [lt=12] too much time used to wait connection release, may be connection leak(used_time_ms=28006832, used connection count=1) [2023-03-27 16:48:01.593888] WARN [SERVER] dump_used_conn_list (ob_inner_sql_connection_pool.cpp:428) [2830644][][T0][Y0-0000000000000000-0-0] [lt=104] ====== DUMP USED CONNECTIONS' BACKTRACE INFO ====== (dump_size=1) [2023-03-27 16:48:01.593943] WARN [SERVER] dump_conn_bt_info (ob_inner_sql_connection.cpp:2395) [2830644][][T0][Y0-0000000000000000-0-0] [lt=12] dump inner sql connection backtrace(tid=2830701, init time="2023-03-27 16:47:32.598193", backtrace="0x3a75987 0x3a755f8 0xb535f71 0xa5c5991 0xa5c3c17 0x3b5025c 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 ") [2023-03-27 16:48:01.612246] WARN [SERVER] alloc_conn (ob_inner_sql_connection_pool.cpp:288) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] connection pool stoped(ret=-8002) [2023-03-27 16:48:01.612293] WARN [SERVER] acquire (ob_inner_sql_connection_pool.cpp:103) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=52] alloc connection from pool failed(ret=-8002) [2023-03-27 16:48:01.612304] WARN [COMMON.MYSQLP] acquire (ob_mysql_proxy.cpp:321) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=9] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:48:01.612314] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:124) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=11] acquire connection failed(ret=-8002, conn=NULL) [2023-03-27 16:48:01.612321] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_operator.cpp:348) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=6] execute sql failed(ret=-8002, sql=update __all_backup_info set value = '' where name = 'backup_scheduler_leader' and value='10.7.206.5:2882') [2023-03-27 16:48:01.612335] WARN [SERVER] clean_backup_scheduler_leader (ob_backup_manager.cpp:517) [2830736][BackupLease][T0][Y0-0000000000000000-0-0] [lt=12] failed to clean backup scheduler leader(ret=-8002) [2023-03-27 16:48:01.640533] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=78] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:01.824595] WARN [SERVER.OMT] get_tenant_base_with_lock (ob_multi_tenant.cpp:150) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=7] get tenant from omt failed(ret=-5150, tenant_id=1) [2023-03-27 16:48:01.824721] WARN [SHARE] switch_to (ob_tenant_base.cpp:489) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=109] switch tenant fail(tenant_id=1, ret=-5150, lbt()="0xb61bbbb 0xa097f8c 0x3a0fa2d 0xa37fac5 0x45e50c5 0x4651b58 0x45e5248 0x3a391f1 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260") [2023-03-27 16:48:01.824739] WARN [SERVER.OMT] operator() (ob_tenant_timezone_mgr.cpp:72) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=13] fail to update time zone info(ret=-5150) [2023-03-27 16:48:01.824760] WARN foreach_refactored (ob_hashtable.h:1318) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=20] fail to do call back, ret=-5150 [2023-03-27 16:48:01.824768] WARN [SERVER.OMT] runTimerTask (ob_tenant_timezone_mgr.cpp:86) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=8] update tenant time zone failed(ret=-5150) [2023-03-27 16:48:01.840844] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=68] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:02.041108] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=60] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:02.241379] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=133] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:02.441560] WARN [SHARE.LOCATION] nonblock_get_leader (ob_ls_location_service.cpp:405) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=43] nonblock get location failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:02.441610] WARN [SHARE.LOCATION] get_leader_with_retry_until_timeout (ob_location_service.cpp:102) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=47] fail to get log stream location leader with retry until_timeout(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", cluster_id=1, tenant_id=1, ls_id={id:1}, leader="0.0.0.0:0", abs_retry_timeout=1679906882599669, retry_interval=200000) [2023-03-27 16:48:02.441624] WARN [SERVER] nonblock_get_leader (ob_inner_sql_connection.cpp:2118) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=13] get leader with retry until timeout failed(ret=-4721, ret="OB_LS_LOCATION_NOT_EXIST", tenant_id=1, ls_id={id:1}, leader="0.0.0.0:0", cluster_id=1, abs_timeout_us=1679906882599669, retry_interval_us=200000) [2023-03-27 16:48:02.441640] WARN [SERVER] execute_write_inner (ob_inner_sql_connection.cpp:1845) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=9] nonblock get leader failed(ret=-4721, tenant_id=1, ls_id={id:1}) [2023-03-27 16:48:02.441743] WARN [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1164) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=33] retry_while_no_tenant_resource failed(ret=-4721, tenant_id=1) [2023-03-27 16:48:02.441752] WARN [SERVER] execute_write (ob_inner_sql_connection.cpp:1786) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=9] execute_write failed(ret=-4721, tenant_id=1, sql=INSERT INTO __all_server_event_history (gmt_create, module, event) VALUES (usec_to_time(1679906852598035), 'observer', 'stop'), is_user_sql=false) [2023-03-27 16:48:02.441764] WARN [SERVER] execute_write (ob_inner_sql_connection.cpp:1775) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=10] execute_write failed(ret=-4721, tenant_id=1, sql="INSERT INTO __all_server_event_history (gmt_create, module, event) VALUES (usec_to_time(1679906852598035), 'observer', 'stop')") [2023-03-27 16:48:02.441773] WARN [COMMON.MYSQLP] write (ob_mysql_proxy.cpp:133) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=7] execute sql failed(ret=-4721, conn=0x7f4746d96050, start=1679906852598165, sql=INSERT INTO __all_server_event_history (gmt_create, module, event) VALUES (usec_to_time(1679906852598035), 'observer', 'stop')) [2023-03-27 16:48:02.441883] WARN [SHARE] process_task (ob_event_history_table_operator.cpp:300) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=9] execute sql failed(sql=INSERT INTO __all_server_event_history (gmt_create, module, event) VALUES (usec_to_time(1679906852598035), 'observer', 'stop'), ret=-4721) [2023-03-27 16:48:02.441901] WARN [SHARE] process (ob_event_history_table_operator.cpp:161) [2830701][EvtHisUpdTask][T0][Y0-0000000000000000-0-0] [lt=18] process_task failed(sql=INSERT INTO __all_server_event_history (gmt_create, module, event) VALUES (usec_to_time(1679906852598035), 'observer', 'stop'), is_delete=false, ret=-4721) [2023-03-27 16:48:02.443247] ERROR [SERVER] main (main.cpp:534) [2830644][][T0][Y0-0000000000000000-0-0] [lt=59] observer start fail(ret=-4009) BACKTRACE:0xb61bbbb 0xb60d4f6 0x3bebb75 0x3beb889 0x3beb6a8 0x3be5eec 0x3be8039 0x3be717c 0x7f4763e4bb47 0x3be5c9e [2023-03-27 16:48:02.456622] WARN [SHARE] add_event (ob_event_history_table_operator.h:373) [2830644][][T0][Y0-0000000000000000-0-0] [lt=121] not init(ret=-4006) [2023-03-27 16:48:02.456676] WARN [SHARE] logical_stop (ob_reentrant_thread.cpp:103) [2830644][][T0][Y0-0000000000000000-0-0] [lt=55] not init(ret=-4006) [2023-03-27 16:48:02.456686] WARN [SHARE] logical_stop (ob_reentrant_thread.cpp:103) [2830644][][T0][Y0-0000000000000000-0-0] [lt=9] not init(ret=-4006) [2023-03-27 16:48:02.456707] WARN [SHARE] logical_wait (ob_reentrant_thread.cpp:118) [2830644][][T0][Y0-0000000000000000-0-0] [lt=5] not init(ret=-4006) [2023-03-27 16:48:02.456714] WARN [SHARE] logical_wait (ob_reentrant_thread.cpp:118) [2830644][][T0][Y0-0000000000000000-0-0] [lt=6] not init(ret=-4006) [2023-03-27 16:48:04.943052] WARN [CLOG] destroy (ob_server_log_block_mgr.cpp:116) [2830644][][T0][Y0-0000000000000000-0-0] [lt=3] ObServerLogBlockMgr destroy(this={dir::"/data/oceanbase/clog/log_pool", dir_fd:14, meta_fd:15, log_pool_meta:{curr_total_size:0, next_total_size:0, status:0}, min_block_id:0, max_block_id:0, is_inited:true}) [2023-03-27 16:48:06.824909] WARN foreach_refactored (ob_hashtable.h:1307) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=6] hashtable not init [2023-03-27 16:48:06.824960] WARN [SERVER.OMT] runTimerTask (ob_tenant_timezone_mgr.cpp:86) [2830652][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=56] update tenant time zone failed(ret=-4006) [2023-03-27 16:48:07.094132] WARN [COMMON] wrlock_all (ob_bucket_lock.cpp:220) [2830644][][T0][Y0-0000000000000000-0-0] [lt=76] The ObBucketLock has not been inited, (ret=-4006) [2023-03-27 16:48:07.098591] WARN [COMMON] ObBucketWLockAllGuard (ob_bucket_lock.h:159) [2830644][][T0][Y0-0000000000000000-0-0] [lt=4456] Fail to try write lock all buckets(ret=-4006) [2023-03-27 16:48:07.098732] WARN begin (ob_hashtable.h:915) [2830644][][T0][Y0-0000000000000000-0-0] [lt=18] hashtable not init, backtrace=0xb61bbbb 0x8fcd2e6 0x8f7cef7 0x8f76b72 0x8f7ccad 0x7f4763e61e61 0x7f4763e61f5a 0x7f4763e4bb4e 0x3be5c9e [2023-03-27 16:48:07.102471] WARN [COMMON] deregister_cache (ob_kv_storecache.cpp:600) [2830644][][T0][Y0-0000000000000000-0-0] [lt=9] The ObKVGlobalCache has not been inited, (ret=-4006) [2023-03-27 16:48:07.102536] WARN [COMMON] deregister_cache (ob_kv_storecache.cpp:600) [2830644][][T0][Y0-0000000000000000-0-0] [lt=69] The ObKVGlobalCache has not been inited, (ret=-4006) [2023-03-27 16:48:07.106693] ERROR [COMMON] reset (ob_fifo_allocator.cpp:200) [2830644][][T0][Y0-0000000000000000-0-0] [lt=4] current_using_ is still used now(ref_count=2, current_using_=0x7f4747024030) BACKTRACE:0xb61bbbb 0xb60d4f6 0x3c12f81 0x3c12c84 0x3c12a99 0x3bf0efb 0x3c05dcd 0x3965191 0x7f4763e61e61 0x7f4763e61f5a 0x7f4763e4bb4e 0x3be5c9e [2023-03-27 16:48:07.106793] WARN foreach_refactored (ob_hashtable.h:1307) [2830644][][T0][Y0-0000000000000000-0-0] [lt=91] hashtable not init [2023-03-27 16:48:07.106799] WARN foreach_refactored (ob_hashtable.h:1307) [2830644][][T0][Y0-0000000000000000-0-0] [lt=6] hashtable not init [2023-03-27 16:48:07.118439] WARN [SERVER] destroy (ob_server.cpp:545) [2830644][][T0][Y0-0000000000000000-0-0] [lt=4] [OBSERVER_NOTICE] can not destroy observer(has_destroy=true, has_stopped=true) [2023-03-27 16:48:07.118515] WARN [CLOG] destroy (ob_server_log_block_mgr.cpp:116) [2830644][][T0][Y0-0000000000000000-0-0] [lt=79] ObServerLogBlockMgr destroy(this={dir::"", dir_fd:14, meta_fd:15, log_pool_meta:{curr_total_size:0, next_total_size:0, status:0}, min_block_id:0, max_block_id:0, is_inited:false}) [2023-03-27 16:48:24.527172] WARN [COMMON] destroy (ob_thread_cond.cpp:65) [2830644][][T0][Y0-0000000000000000-0-0] [lt=23] Fail to destroy pthread mutex, (ret=16) [2023-03-27 16:48:24.727648] WARN [COMMON] unlock (ob_thread_cond.h:60) [2830762][][T0][Y0-0000000000000000-0-0] [lt=0] The thread cond has not been inited, (ret=-4006, lbt()=0xb61bbbb 0x3c50038 0x3afdb71 0x5c309e9 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260) [2023-03-27 16:48:24.727712] ERROR [COMMON] ~ObLockGuard (ob_lock_guard.h:60) [2830762][][T0][Y0-0000000000000000-0-0] [lt=69] Fail to unlock, (ret=-4006) BACKTRACE:0xb61bbbb 0xb60d4f6 0x3bebb75 0x3beb889 0x3beb6a8 0x3be5eec 0x3c4ff5d 0x3afdb86 0x5c309e9 0xb5fc3ac 0xb5fa7ea 0x7f4763fe8f1b 0x7f4763f1e260 [2023-03-27 16:48:24.728113] WARN [COMMON] deregister_cache (ob_kv_storecache.cpp:600) [2830644][][T0][Y0-0000000000000000-0-0] [lt=4746] The ObKVGlobalCache has not been inited, (ret=-4006) [2023-03-27 16:48:24.774985] ERROR has_unfree_callback (object_set.cpp:34) [2830644][][T0][Y0-0000000000000000-0-0] [lt=90] HAS UNFREE PTR!!! context: 0x7f47479fe6b0, label: SchemaSysCache, static_id: 0x117e2098, static_info:{filename: ob_schema_cache.cpp, line: 467, function: init}, dynamic_info:{tid: 2830644, cid: 0, create_time: 1679906852054289} BACKTRACE:0xb61bbbb 0xb60d4f6 0x3be889a 0x3be8577 0x3be8361 0x3bfad18 0x3bfac8d 0x394e45e 0x39057a3 0x3bef76b 0x3905cb8 0x990189b 0x9858752 0x7f4763e61e61 0x7f4763e61f5a 0x7f4763e4bb4e 0x3be5c9e [2023-03-27 16:48:24.775199] WARN [COMMON] deregister_cache (ob_kv_storecache.cpp:600) [2830644][][T0][Y0-0000000000000000-0-0] [lt=205] The ObKVGlobalCache has not been inited, (ret=-4006) [2023-03-27 16:48:24.775229] WARN [COMMON] deregister_cache (ob_kv_storecache.cpp:600) [2830644][][T0][Y0-0000000000000000-0-0] [lt=10] The ObKVGlobalCache has not been inited, (ret=-4006)