[2023-11-27 10:40:46.236172] WARN [SQL.PC] common_free (ob_lib_cache_object_manager.cpp:139) [69238][T1002_TNT_L5_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=22] set logical del time(cache_obj->get_logical_del_time()=506888307666, cache_obj->added_lc()=false, cache_obj->get_object_id()=354, cache_obj->get_tenant_id()=1002, lbt()="0xb61bbbb 0x62c14bd 0x391b3dd 0x391aac9 0x3af7508 0x3ebbc5f 0x3eba8bd 0x3b420f3 0x3ea0c79 0x3b8af35 0x390ab41 0x45eb379 0x3902f24 0x45eb917 0xb5ffbb7 0xb5fa7ea 0x7fddb6e0eea5 0x7fddb6b37b0d") [2023-11-27 10:40:46.239409] WARN [SQL.PC] common_free (ob_lib_cache_object_manager.cpp:139) [69238][T1002_TNT_L5_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=21] set logical del time(cache_obj->get_logical_del_time()=506888310904, cache_obj->added_lc()=false, cache_obj->get_object_id()=355, cache_obj->get_tenant_id()=1002, lbt()="0xb61bbbb 0x62c14bd 0x391b3dd 0x391aac9 0x3af7508 0x3ebbc5f 0x3eba8bd 0x3b420f3 0x3ea0c79 0x3b8af35 0x390ab41 0x45eb379 0x3902f24 0x45eb917 0xb5ffbb7 0xb5fa7ea 0x7fddb6e0eea5 0x7fddb6b37b0d") [2023-11-27 10:40:46.240422] INFO [LIB] get_instance (ob_tsi_factory.h:372) [69253][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=16] new instance succ [N9oceanbase6common13CStringBufMgrE] 0x7fdcfbb6f800 size=24, tsi=0x7fdd3fdf6560 [2023-11-27 10:40:46.240454] INFO [LIB] get_instance (ob_tsi_factory.h:372) [69253][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=20] new instance succ [N9oceanbase6common13CStringBufMgr8BufArrayE] 0x7fdc705ec050 size=61520, tsi=0x7fdd3fdf6560 [2023-11-27 10:40:46.245230] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=15] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.245275] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=44] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.245296] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=20] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.245309] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=12] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.245322] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=12] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.247206] INFO [STORAGE] add_memtable_ (ob_i_memtable_mgr.cpp:348) [69238][T1002_TNT_L5_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=26] succeed to add memtable(get_memtable_count_()=1, memtable_handle={table:0x7fdccc3722d0, t3m_:0x7fdd03dd4030, allocator_:null, table_type_:0}, lbt()="0xb61bbbb 0x883fc65 0x883f70c 0x955f098 0x94f378b 0x95119f3 0x898812a 0x39f27ec 0x3a6a859 0x3a69f57 0x3a0ca19 0x3a0be16 0x3a0a55c 0x3a0a0f5 0x3a084fb 0x3a082e4 0x3946de7 0x71c69f4 0x3a08221 0x7188e54 0x39468c7 0x70fc2eb 0x3922119 0x39215e0 0x394797b 0x3945b41 0x3b0b08b 0x3b0abd0 0x3b098e0 0x3ebaf24 0x3eba8bd 0x3b420f3 0x3ea0c79 0x3b8af35 0x390ab41 0x45eb379 0x3902f24 0x45eb917 0xb5ffbb7 0xb5fa7ea 0x7fddb6e0eea5 0x7fddb6b37b0d") [2023-11-27 10:40:46.247221] INFO [STORAGE] create_memtable (ob_tablet_memtable_mgr.cpp:249) [69238][T1002_TNT_L5_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=15] succeed to create memtable(ret=0, ls_id={id:1}, memtable={ObITable:{this:0x7fdccc3722d0, key:{tablet_id:{id:208}, column_group_idx:0, table_type:"MEMTABLE", log_ts_range:{start_log_ts:1700742270217437247, end_log_ts:9223372036854775807}}, ref_cnt:2, upper_trans_version:9223372036854775807, timestamp:1701052846247187}, this:0x7fdccc3722d0, timestamp:1701052846247187, state:0, freeze_clock:0, max_schema_version:0, write_ref_cnt:0, local_allocator:{ListHandle:{freeze_stat:0, id:-1, clock:9223372036854775807}, host:0x7fdd627cc030, arena_handle:{allocated:0}, last_freeze_timestamp:1701052322457926}, unsubmitted_cnt:0, unsynced_cnt:0, logging_blocked:false, unset_active_memtable_logging_blocked:false, resolve_active_memtable_left_boundary:true, contain_hotspot_row:false, max_end_log_ts:0, rec_log_ts:9223372036854775807, snapshot_version:9223372036854775807, migration_clog_checkpoint_ts:0, is_tablet_freeze:false, is_force_freeze:false, contain_hotspot_row:false, read_barrier:false, is_flushed:false, freeze_state:0, mt_stat_.frozen_time:0, mt_stat_.ready_for_flush_time:0, mt_stat_.create_flush_dag_time:0, mt_stat_.release_time:0}) [2023-11-27 10:40:46.247253] INFO [STORAGE] inner_create_memtable (ob_tablet.cpp:1651) [69238][T1002_TNT_L5_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=31] succeeded to create memtable for tablet(ret=0, tablet_meta_={version:1, ls_id:{id:1}, tablet_id:{id:208}, data_tablet_id:{id:208}, ref_tablet_id:{id:0}, create_scn:1677229237095064373, start_scn:1699511759130056712, clog_checkpoint_ts:1700742270217437247, ddl_checkpoint_ts:1, snapshot_version:1700742270101923007, multi_version_start:1700740459329142561, autoinc_seq:{intervals:[]}, compat_mode:0, ha_status:{restore_status:0, data_status:0, expected_status:0, reserved:0}, report_status:{merge_snapshot_version:1700330403040590253, cur_report_version:1700330403040590253, data_checksum:3750726089, row_count:1724}, tx_data:{version:1, length:24, tx_id:{txid:0}, tx_log_ts:1677229237262902262, tablet_status:{status:1}, is_tx_end:false, unsynced_cnt_for_multi_data:0}, ddl_data:{redefined:false, snapshot_version:-1, schema_version:-1, data_tablet_id:{id:0}, hidden_tablet_ids:[], lob_meta_tablet_id:{id:50208}, lob_piece_tablet_id:{id:60208}, is_tx_end:false, unsynced_cnt_for_multi_data:0}, table_store_flag:{with_major_sstable:1}, ddl_start_log_ts:0, ddl_snapshot_version:0, max_sync_storage_schema_version:1677229238987640, ddl_execution_id:0, ddl_cluster_version:0}, tablet_id={id:208}, clog_checkpoint_ts=1700742270217437247, schema_version=0) [2023-11-27 10:40:46.256791] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=18] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.256813] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=22] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.256830] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=16] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.256842] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=11] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.256856] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=13] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.256876] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=10] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.256884] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=8] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.256892] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=8] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.256899] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.256906] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.256917] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=8] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.256924] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.256932] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=8] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.256939] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.256945] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.256956] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.256962] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.256970] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.256976] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.256983] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.256993] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257000] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257007] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.257013] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=5] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257019] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257035] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257042] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257050] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.257055] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257062] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257071] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257077] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257084] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.257090] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=5] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257097] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257105] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257112] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257119] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.257124] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257131] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257139] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257146] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257153] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=8] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.257159] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257167] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257202] WARN [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:630) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=30] can not find enough memory block to wash(ret=-4273, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257209] INFO [COMMON] try_flush_washable_mb (ob_kvcache_store.cpp:648) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=8] ObKVCache try flush washable memblock details(ret=-4273, tenant_id=1001, cache_id=-1, size_washed=0, size_need_washed=2097152) [2023-11-27 10:40:46.257217] WARN [COMMON] sync_wash_mbs (ob_kvcache_store.cpp:536) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] Fail to try flush mb(ret=-4273, tenant_id=1001) [2023-11-27 10:40:46.257223] WARN [COMMON] sync_wash_mbs (ob_kv_storecache.cpp:851) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257229] WARN [COMMON] alloc_chunk (ob_resource_mgr.cpp:94) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sync_wash_mbs failed(ret=-4273, tenant_id=1001, wash_size=2097152, wash_single_mb=true) [2023-11-27 10:40:46.257238] WARN alloc_new_page (page_arena.h:330) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] cannot allocate memory.sz=20720, pages_=0,total_=0 [2023-11-27 10:40:46.257245] WARN alloc (ob_lib_cache_object_manager.h:134) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] failed to allocate memory for lib cache node(ret=-4013) [2023-11-27 10:40:46.257253] WARN [SQL.PC] alloc (ob_lib_cache_object_manager.cpp:72) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] failed to create lib cache node(ret=-4013, ns=1) [2023-11-27 10:40:46.257264] WARN [SQL.PC] alloc_cache_obj (ob_plan_cache.cpp:1938) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=8] failed to alloc cache obj(ret=-4013) [2023-11-27 10:40:46.257271] WARN [SQL.PC] alloc (ob_cache_object_factory.cpp:43) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] failed to alloc cache obj(ret=-4013, ns=1) [2023-11-27 10:40:46.257279] WARN [SQL] generate_physical_plan (ob_sql.cpp:2401) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] fail to alloc phy_plan(ret=-4013) [2023-11-27 10:40:46.257291] WARN [SQL] handle_physical_plan (ob_sql.cpp:3779) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] Failed to generate plan(ret=-4013, result.get_exec_context().need_disconnect()=false) [2023-11-27 10:40:46.257299] WARN [SQL] handle_text_query (ob_sql.cpp:1917) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] fail to handle physical plan(ret=-4013) [2023-11-27 10:40:46.257307] WARN [SQL] stmt_query (ob_sql.cpp:175) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] fail to handle text query(stmt=SELECT * FROM __all_balance_group_ls_stat WHERE tenant_id = 1002 AND balance_group_id_high = 1002 AND balance_group_id_low = 0 FOR UPDATE, ret=-4013) [2023-11-27 10:40:46.257318] WARN [SERVER] do_query (ob_inner_sql_connection.cpp:595) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=8] executor execute failed(ret=-4013) [2023-11-27 10:40:46.257324] WARN [SERVER] query (ob_inner_sql_connection.cpp:733) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] execute failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_balance_group_ls_stat WHERE tenant_id = 1002 AND balance_group_id_high = 1002 AND balance_group_id_low = 0 FOR UPDATE"}, retry_cnt=0) [2023-11-27 10:40:46.257336] WARN [SERVER] after_func (ob_query_retry_ctrl.cpp:830) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=10] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2023-11-27 10:40:46.257351] WARN [SERVER] inner_close (ob_inner_sql_result.cpp:211) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=13] result set close failed(ret=-4013) [2023-11-27 10:40:46.257358] WARN [SERVER] force_close (ob_inner_sql_result.cpp:191) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] result set close failed(ret=-4013) [2023-11-27 10:40:46.257364] WARN [SERVER] query (ob_inner_sql_connection.cpp:738) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=5] failed to close result(close_ret=-4013, ret=-4013) [2023-11-27 10:40:46.257378] WARN [SERVER] query (ob_inner_sql_connection.cpp:763) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_balance_group_ls_stat WHERE tenant_id = 1002 AND balance_group_id_high = 1002 AND balance_group_id_low = 0 FOR UPDATE"}, record_ret=-4013, ret=-4013) [2023-11-27 10:40:46.257397] WARN [SERVER] query (ob_inner_sql_connection.cpp:780) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=16] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_balance_group_ls_stat WHERE tenant_id = 1002 AND balance_group_id_high = 1002 AND balance_group_id_low = 0 FOR UPDATE"}, aret=-4013, ret=-4013) [2023-11-27 10:40:46.257408] WARN [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:2003) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=10] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_balance_group_ls_stat WHERE tenant_id = 1002 AND balance_group_id_high = 1002 AND balance_group_id_low = 0 FOR UPDATE) [2023-11-27 10:40:46.257416] WARN [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1164) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2023-11-27 10:40:46.257424] WARN [SERVER] execute_read (ob_inner_sql_connection.cpp:1943) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] execute_read failed(ret=-4013, cluster_id=1, tenant_id=1001) [2023-11-27 10:40:46.257432] WARN [SERVER] process_read (ob_inner_sql_rpc_processor.cpp:128) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] failed to exec read_sql(ret=-4013, read_sql=SELECT * FROM __all_balance_group_ls_stat WHERE tenant_id = 1002 AND balance_group_id_high = 1002 AND balance_group_id_low = 0 FOR UPDATE) [2023-11-27 10:40:46.257462] WARN [SERVER] process (ob_inner_sql_rpc_processor.cpp:340) [69086][T1001_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=8] process_read failed(ret=-4013, sql_str=SELECT * FROM __all_balance_group_ls_stat WHERE tenant_id = 1002 AND balance_group_id_high = 1002 AND balance_group_id_low = 0 FOR UPDATE) [2023-11-27 10:40:46.258590] INFO [STORAGE.TRANS] rollback_tx (ob_tx_api.cpp:287) [68954][T1_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=109] rollback tx(ret=0, *this={is_inited_:true, tenant_id_:1001, this:0x7fdd23004030}, tx={this:0x7fdcbb34ed40, tx_id:{txid:735204598}, state:7, addr:"192.168.12.123:2882", tenant_id:1001, session_id:1, xid:NULL, access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:-1, snapshot_scn:0, active_scn:1701052846243015, op_sn:3, alloc_ts:1701052846245171, active_ts:1701052846245171, commit_ts:-1, finish_ts:1701052846258042, timeout_us:999932197, lock_timeout_us:-1, expire_ts:1701053846177368, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:-1, commit_cb:null, cluster_id:1, cluster_version:17179869184, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:-6002, commit_expire_ts:0, commit_task_.is_registered():false, ref:2}) [2023-11-27 10:40:46.258699] TRACE [TRACE] reset (ob_trans_define_v4.cpp:295) [68954][T1_TNT_L5_G0][T1001][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=103] [tx desc trace](TRACE=begin_ts=1701052846245484 2023-11-27 02:40:46.245484|[acquire] u=0 ret:0, addr:0x7fdcbb34ed40, session:1, ref:1, thread_id:69086|[start_tx] u=6 ret:0, txid:{txid:735204598}, isolation_level:1, ref:2, thread_id:69086|[rollback_tx] u=13199 ret:0, ref:2, thread_id:68954|total_timeu=13205) [2023-11-27 10:40:46.260201] INFO [STORAGE.TRANS] abort_ (ob_trans_part_ctx.cpp:1595) [68954][T1_TNT_L5_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=82] tx abort(ret=0, reason=-6002, reason=-6002, reason="OB_TRANS_ROLLBACKED", this={this:0x7fdcd6025a10, trans_id:{txid:52000168}, tenant_id:1002, is_exiting:true, trans_expired_time:1701053846177049, cluster_version:17179869184, trans_need_wait_wrap:{receive_gts_ts:{mts:0}, need_wait_interval_us:0}, stc:{mts:0}, ctx_create_time:1701052846221326}{ls_id:{id:1}, session_id:1, part_trans_action:4, pending_write:0, exec_info:{state:60, upstream:{id:-1}, participants:[], incremental_participants:[], prev_record_lsn:{lsn:18446744073709551615}, redo_lsns:[], redo_log_no:0, multi_data_source:[], scheduler:"192.168.12.123:2882", prepare_version:-1, trans_type:0, next_log_entry_no:0, max_applied_log_ts:-1, max_applying_log_ts:-1, max_applying_part_log_no:9223372036854775807, max_submitted_seq_no:0, checksum:0, checksum_log_ts:0, max_durable_lsn:{lsn:18446744073709551615}, data_complete:false, is_dup_tx:false, prepare_log_info_arr:[], xid:{gtrid_str:"", bqual_str:"", format_id:1, gtrid_str_.ptr():"data_size:0, data:", bqual_str_.ptr():"data_size:0, data:", g_hv:0, b_hv:0}, need_checksum:true}, sub_state:{flag:0}, is_leaf():false, is_root():true, busy_cbs_.get_size():0, final_log_cb_:{ObTxBaseLogCb:{log_ts:0, lsn:{lsn:18446744073709551615}, submit_ts:0}, this:0x7fdcd6029bc0, is_inited_:true, trans_id:{txid:52000168}, ls_id:{id:1}, ctx:0x7fdcd6025a10, tx_data:null, is_callbacked_:false, mds_range_:{count_:0}, cb_arg_array_:[]}, ctx_tx_data_:{ctx_mgr_:0x7fdcdb004030, tx_data_:{tx_id:{txid:52000168}, state:"ABORT", in_tx_data_table:"False", commit_version:-1, start_log_ts:9223372036854775807, end_log_ts:9223372036854775807, undo_status_list:{head:null, undo_node_cnt:0}}, tx_commit_data_:{tx_id:{txid:9223372036854775807}, state:0, is_in_tx_data_table:false, commit_version:-1, start_log_ts:9223372036854775807, end_log_ts:9223372036854775807}, read_only_:false}, role_state_:0, start_replay_ts_:0, is_incomplete_replay_ctx_:false, mt_ctx_:{ObIMvccCtx={alloc_type=0 ctx_descriptor=0 min_table_version=9223372036854775807 max_table_version=0 trans_version=9223372036854775807 commit_version=-1 row_purge_version=0 lock_wait_start_ts=0 replay_compact_version=0} end_code=-6002 tx_status=1 is_readonly=false ref=0 trans_id={txid:52000168} ls_id=1 callback_alloc_count=18 callback_free_count=18 checksum=0 tmp_checksum=0 checksum_log_ts=0 redo_filled_count=0 redo_sync_succ_count=0 redo_sync_fail_count=0 main_list_length=0 unsynced_cnt=0 unsubmitted_cnt_=0 cb_statistics:[main=31, slave=0, merge=0, tx_end=31, rollback_to=0, fast_commit=0, remove_memtable=0]}, coord_prepare_info_arr_:[], upstream_state:10, retain_cause:-1, collected:[], ref:1, rec_log_ts:-1, prev_rec_log_ts:-1}) [2023-11-27 10:40:46.260471] INFO [STORAGE.TRANS] handle_trans_abort_request (ob_trans_service_v4.cpp:1682) [68954][T1_TNT_L5_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=252] handle trans abort request(ret=0, abort_req={txMsg:{type:22, cluster_version:17179869184, tenant_id:1002, tx_id:{txid:52000168}, receiver:{id:1}, sender:{id:9223372036854775807}, sender_addr:"192.168.12.123:2882", epoch:-1, request_id:10, timestamp:1701052846259806, cluster_id:1}, reason:-6002}) [2023-11-27 10:40:46.260568] INFO [STORAGE.TRANS] rollback_tx (ob_tx_api.cpp:287) [68954][T1_TNT_L5_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=92] rollback tx(ret=0, *this={is_inited_:true, tenant_id_:1002, this:0x7fdcfb804030}, tx={this:0x7fdcab7c6ec0, tx_id:{txid:52000168}, state:7, addr:"192.168.12.123:2882", tenant_id:1002, session_id:1, xid:NULL, access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:-1, snapshot_scn:0, active_scn:1701052846218060, op_sn:10, alloc_ts:1701052846219137, active_ts:1701052846219137, commit_ts:-1, finish_ts:1701052846260198, timeout_us:999957912, lock_timeout_us:-1, expire_ts:1701053846177049, coord_id:{id:-1}, parts:[{id:{id:1}, addr:"192.168.12.123:2882", epoch:137, first_scn:1701052846220225, last_scn:1701052846245173, last_touch_ts:1701052846220226}], exec_info_reap_ts:1701052846220225, commit_version:-1, commit_cb:null, cluster_id:1, cluster_version:17179869184, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:-6002, commit_expire_ts:0, commit_task_.is_registered():false, ref:2}) [2023-11-27 10:40:46.260652] TRACE [TRACE] reset (ob_trans_define_v4.cpp:295) [68954][T1_TNT_L5_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=78] [tx desc trace](TRACE=begin_ts=1701052846219864 2023-11-27 02:40:46.219864|[acquire] u=0 ret:0, addr:0x7fdcab7c6ec0, session:1, ref:1, thread_id:69238|[start_tx] u=7 ret:0, txid:{txid:52000168}, isolation_level:1, ref:2, thread_id:69238|[create_global_implicit_savepoint] u=1149 ret:0, txid:{txid:52000168}, savepoint:1701052846220224, release:true, opid:3, ref:2, thread_id:69238|[get_read_snapshot] u=13 ret:0, expire_ts:1701053846107610, isolation_level:1, snapshot_source:1, snapshot_version:1701052846221028069, snapshot_txid:52000168, snapshot_scn:1701052846220224, trace_id:"YB42C0A80C7B-00060B1921AAEBBB-0-0", ref:2, thread_id:69238|[create_global_implicit_savepoint] u=8433 ret:0, txid:{txid:52000168}, savepoint:1701052846228929, release:true, opid:4, ref:2, thread_id:69238|[create_global_implicit_savepoint] u=1404 ret:0, txid:{txid:52000168}, savepoint:1701052846230027, release:true, opid:5, ref:2, thread_id:69238|[create_global_implicit_savepoint] u=5000 ret:0, txid:{txid:52000168}, savepoint:1701052846232221, release:true, opid:6, ref:2, thread_id:69238|[create_global_implicit_savepoint] u=3359 ret:0, txid:{txid:52000168}, savepoint:1701052846236528, release:true, opid:7, ref:2, thread_id:69238|[create_global_implicit_savepoint] u=1130 ret:0, txid:{txid:52000168}, savepoint:1701052846239754, release:true, opid:8, ref:2, thread_id:69253|[create_global_implicit_savepoint] u=6743 ret:0, txid:{txid:52000168}, savepoint:1701052846245172, release:true, opid:9, ref:2, thread_id:69238|[rollback_tx] u=13543 ret:0, ref:2, thread_id:68954|total_timeu=40781) [2023-11-27 10:40:46.264949] WARN [RPC.OBRPC] rpc_call (ob_rpc_proxy.ipp:367) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=22] execute rpc fail(ret=-4013, dst="192.168.12.121:2882") [2023-11-27 10:40:46.264974] WARN log_user_error_and_warn (ob_rpc_proxy.cpp:320) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=24] [2023-11-27 10:40:46.264993] WARN [SQL.ENG] execute (ob_table_executor.cpp:452) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=9] rpc proxy create table failed(ret=-4013, dst="192.168.12.121:2882") [2023-11-27 10:40:46.265064] INFO [SHARE] add_event (ob_event_history_table_operator.h:268) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=8] event table add task(ret=0, event_table_name="__all_server_event_history", sql=INSERT INTO __all_server_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, svr_ip, svr_port) VALUES (usec_to_time(1701052846265003), 'sql', 'execute_cmd', 'cmd_type', 20, 'sql_text', X'435245415445205441424C45207461626C653128696420696E7429', 'return_code', -4013, '192.168.12.123', 2882)) [2023-11-27 10:40:46.265089] WARN [SQL] open_cmd (ob_result_set.cpp:94) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=18] execute cmd failed(ret=-4013) [2023-11-27 10:40:46.265099] WARN [SQL] open (ob_result_set.cpp:150) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=9] execute plan failed(ret=-4013) [2023-11-27 10:40:46.265109] WARN [SERVER] response_result (ob_sync_cmd_driver.cpp:117) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=5] close result set fail(cret=-4013) [2023-11-27 10:40:46.265115] WARN [SERVER] after_func (ob_query_retry_ctrl.cpp:830) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=5] [RETRY] check if need retry(v={force_local_retry:false, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2023-11-27 10:40:46.265125] WARN [SERVER] response_result (ob_sync_cmd_driver.cpp:123) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=11] result set open failed, check if need retry(ret=-4013, cli_ret=-4013, retry_ctrl_.need_retry()=0) [2023-11-27 10:40:46.265142] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:306) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=6] sending error packet(err=-4013, extra_err_info=NULL, lbt()="0xb61bbbb 0x581edce 0x57d9c16 0x39e0cb1 0x39137eb 0x390eb08 0x390c66a 0x390ab41 0x45eb379 0x3902f24 0x45eb917 0xb5ffbb7 0xb5fa7ea 0x7fddb6e0eea5 0x7fddb6b37b0d") [2023-11-27 10:40:46.265194] WARN [SERVER] do_process (obmp_query.cpp:815) [69246][T1002_TNT_L0_G0][T1002][YB42C0A80C7B-00060B1921AAEBBB-0-0] [lt=7] execute query fail(ret=-4013, timeout_timestamp=1701053846177078)