内存一直不释放

淇铭 · 2025 年6 月 4 日 18:28

select * from oceanbase.__all_virtual_malloc_sample_info where mod_name=‘xxx’ order by alloc_size desc limit 10;
–获取到backtrace信息
addr2line -pCfe backtrace
–用addr2line打一下堆栈信息看看分析一下看看是否有问题

sunnyli · 2025 年6 月 4 日 18:40

obclient [(none)]> select * from oceanbase.__all_virtual_malloc_sample_info where mod_name=‘IoControl’ order by alloc_bytes desc limit 10;
±--------------±---------±----------±-------±----------±----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------±---------------±------------±------------+
| svr_ip | svr_port | tenant_id | ctx_id | mod_name | back_trace | ctx_name | alloc_count | alloc_bytes |
±--------------±---------±----------±-------±----------±----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------±---------------±------------±------------+
| 192.168.0.175 | 2882 | 1004 | 0 | IoControl | 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x117cd1dc 0x16ca0d20 0x16c927d0 0x16c91f4c 0x16983380 0x16982de4 0xb45d7c8 0xb45b704 0x1b956a10 0x1b954c24 0xffff9e1b00e4 | DEFAULT_CTX_ID | 209 | 14025866272 |
| 192.168.0.175 | 2882 | 1004 | 0 | IoControl | 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x117cd1dc 0x16ca0d20 0x16c927d0 0x16c91f4c 0x16983380 0x16983d44 0xb45d87c 0xb45b704 0x1b956a10 0x1b954c24 0xffff9e1b00e4 | DEFAULT_CTX_ID | 57 | 1341144336 |
| 192.168.0.175 | 2882 | 1004 | 0 | IoControl | 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x16c8f77c 0x16c99ae8 0x1714ab84 0xba6fb24 0x14bd9e08 0x14bd95e8 0xc5f0f38 0x88d86f8 0x0 0x0 0x0 | DEFAULT_CTX_ID | 1 | 15600544 |
| 192.168.0.175 | 2882 | 1003 | 0 | IoControl | 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x16c8f77c 0x16c99ae8 0x1714ab84 0xba6fb24 0x14bd9e08 0x14bd95e8 0xc5f0f38 0x88d86f8 0x0 0x0 0x0 | DEFAULT_CTX_ID | 1 | 15600544 |
| 192.168.0.175 | 2882 | 1 | 0 | IoControl | 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x16c8f77c 0x16c99ae8 0x1714ab84 0xba6fb24 0x14bd9e08 0x14bd95e8 0xc5f0f38 0x88d86f8 0x0 0x0 0x0 | DEFAULT_CTX_ID | 1 | 15600544 |
| 192.168.0.175 | 2882 | 1002 | 0 | IoControl | 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x16c8f77c 0x16c99ae8 0x1714ab84 0xba6fb24 0x14bd9e08 0x14bd95e8 0xc5f0f38 0x88d86f8 0x0 0x0 0x0 | DEFAULT_CTX_ID | 1 | 15600544 |
| 192.168.0.175 | 2882 | 500 | 0 | IoControl | 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x16c8f77c 0x16c8d41c 0xc5d63ac 0x88d86e4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 | DEFAULT_CTX_ID | 1 | 15600544 |
| 192.168.0.175 | 2882 | 1001 | 0 | IoControl | 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x16c8f77c 0x16c99ae8 0x1714ab84 0xba6fb24 0x14bd9e08 0x14bd95e8 0xc5f0f38 0x88d86f8 0x0 0x0 0x0 | DEFAULT_CTX_ID | 1 | 15600544 |
| 192.168.0.175 | 2882 | 500 | 0 | IoControl | 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x16c8f59c 0x16c8d41c 0xc5d63ac 0x88d86e4 0x0 0x0 0x0 0x0 0x0 0x0 0x0 | DEFAULT_CTX_ID | 1 | 9600544 |
| 192.168.0.175 | 2882 | 1004 | 0 | IoControl | 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x16c8f59c 0x16c99ae8 0x1714ab84 0xba6fb24 0x14bd9e08 0x14bd95e8 0xc5f0f38 0x88d86f8 0x0 0x0 0x0 | DEFAULT_CTX_ID | 1 | 9600544 |
±--------------±---------±----------±-------±----------±----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------±---------------±------------±------------+
10 rows in set (0.023 sec)
back_trace怎么那么多，怎么办

collapsar · 2025 年6 月 4 日 19:10

addr2line -pCfe bin/observer 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x117cd1dc 0x16ca0d20 0x16c927d0 0x16c91f4c 0x16983380 0x16982de4 0xb45d7c8 0xb45b704 0x1b956a10 0x1b954c24 0xffff9e1b00e4

找一个看一下

靖顺 · 2025 年6 月 4 日 20:20

综合看下来，IoControl 这个模块内存长期被占用，这里有一篇排查知识库，你可以看看

https://www.oceanbase.com/knowledge-base/oceanbase-database-1000000000624471?back=kb

sunnyli · 2025 年6 月 5 日 09:04

obclient [oceanbase]> SELECT * FROM __all_virtual_mem_leak_checker_info ORDER BY alloc_size DESC LIMIT 5;
±--------------±---------±----------±---------±------------±-----------±---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| svr_ip | svr_port | mod_name | mod_type | alloc_count | alloc_size | back_trace |
±--------------±---------±----------±---------±------------±-----------±---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 192.168.0.175 | 2882 | IoControl | user | 73 | 4898986784 | 0x1be3362c 0x11408fd8 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x117cd1dc 0x16ca0d20 0x16c927d0 0x16c91f4c 0x16983380 0x16982de4 0xb45d7c8 0xb45b704 0x1b956a10 0x1b954c24 0xffff9e1b00e4 |
| 192.168.0.175 | 2882 | IoControl | user | 18 | 423519264 | 0x1be3362c 0x11408fd8 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x117cd1dc 0x16ca0d20 0x16c927d0 0x16c91f4c 0x16983380 0x16983d44 0xb45d87c 0xb45b704 0x1b956a10 0x1b954c24 0xffff9e1b00e4 |
±--------------±---------±----------±---------±------------±-----------±---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
2 rows in set (0.001 sec) alloc_count=73算大么
IO STATUS SENDER.txt (96.2 KB)

sunnyli · 2025 年6 月 5 日 09:46

怎么使用addr2line打印，addr2line -e <file_path> 中file_path使用哪个？？

淇铭 · 2025 年6 月 5 日 10:04

找第一个就可以 ps -ef ｜ grep observer | grep -v grep 可以查看一下 observer进程在那个目录下按照下面的方式执行就好了
addr2line -pCfe bin/observer 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x117cd1dc 0x16ca0d20 0x16c927d0 0x16c91f4c 0x16983380 0x16982de4 0xb45d7c8 0xb45b704 0x1b956a10 0x1b954c24 0xffff9e1b00e4

sunnyli · 2025 年6 月 5 日 10:09

[root@pdyw log]# addr2line -pCfe /home/hyzn/admin/myoceanbase/oceanbase/bin/observer 0x1be3362c 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x117cd1dc 0x16ca0d20 0x16c927d0 0x16c91f4c 0x16983380 0x16982de4 0xb45d7c8 0xb45b704 0x1b956a10 0x1b954c24 0xffff9e1b00e4
oceanbase::common::light_backtrace(void**, int, long) at ??:?
oceanbase::lib::ObMallocAllocator::alloc(long, oceanbase::lib::ObMemAttr const&) at ??:?
oceanbase::common::ObBlockAllocMgr::alloc_block(long, oceanbase::lib::ObMemAttr&) at ??:?
oceanbase::common::ObVSliceAlloc::alloc(long) at 0_cxx.cxx:?
oceanbase::common::ObIOAllocator::alloc(long) at 0_cxx.cxx:?
oceanbase::common::ObIORequest::alloc_aligned_io_buf(char*&) at 0_cxx.cxx:?
oceanbase::common::ObTenantIOManager::alloc_req_and_result(oceanbase::common::ObSNIOInfo const&, oceanbase::common::ObIOHandle&, oceanbase::common::ObIORequest*&, oceanbase::common::ObRefHolderoceanbase::common::ObIORequest&) at 0_cxx.cxx:?
oceanbase::common::ObIOManager::tenant_aio(oceanbase::common::ObSNIOInfo const&, oceanbase::common::ObIOHandle&) at 0_cxx.cxx:?
oceanbase::common::ObIOManager::aio_write(oceanbase::common::ObSNIOInfo const&, oceanbase::common::ObIOHandle&) at ??:?
oceanbase::common::ObBackupIoAdapter::io_manager_write(char const*, long, long, oceanbase::common::ObIOFd const&, long&) at ??:?
oceanbase::common::ObBackupIoAdapter::write_single_file(oceanbase::common::ObString const&, oceanbase::share::ObBackupStorageInfo const*, char const*, long, oceanbase::common::ObStorageIdMod const&) at ??:?
oceanbase::archive::ObArchiveSender::handle(oceanbase::archive::ObArchiveSendTask&, oceanbase::archive::ObArchiveSender::TaskConsumeStatus&) at 0_cxx.cxx:?
oceanbase::archive::ObArchiveSender::run1() at 0_cxx.cxx:?
oceanbase::lib::Threads::run(long) at ??:?
oceanbase::lib::__th_start(void*) at 0_cxx.cxx:?
?? ??:0

[root@pdyw log]# addr2line -pCfe /home/hyzn/admin/myoceanbase/oceanbase/bin/observer 0x1be3362c 0x11408fd8 0x113f8f94 0x11458af8 0x1143f930 0x117cd000 0x117cd1dc 0x16ca0d20 0x16c927d0 0x16c91f4c 0x16983380 0x16982de4 0xb45d7c8 0xb45b704 0x1b956a10 0x1b954c24 0xffff9e1b00e4
oceanbase::common::light_backtrace(void**, int, long) at ??:?
void* oceanbase::lib::ObTenantCtxAllocator::common_reallococeanbase::lib::ObjectMgr(void const*, long, oceanbase::lib::ObMemAttr const&, oceanbase::lib::ObTenantCtxAllocator&, oceanbase::lib::ObjectMgr&) at ??:?
oceanbase::lib::ObMallocAllocator::alloc(long, oceanbase::lib::ObMemAttr const&) at ??:?
oceanbase::common::ObBlockAllocMgr::alloc_block(long, oceanbase::lib::ObMemAttr&) at ??:?
oceanbase::common::ObVSliceAlloc::alloc(long) at 0_cxx.cxx:?
oceanbase::common::ObIOAllocator::alloc(long) at 0_cxx.cxx:?
oceanbase::common::ObIORequest::alloc_aligned_io_buf(char*&) at 0_cxx.cxx:?
oceanbase::common::ObTenantIOManager::alloc_req_and_result(oceanbase::common::ObSNIOInfo const&, oceanbase::common::ObIOHandle&, oceanbase::common::ObIORequest*&, oceanbase::common::ObRefHolderoceanbase::common::ObIORequest&) at 0_cxx.cxx:?
oceanbase::common::ObIOManager::tenant_aio(oceanbase::common::ObSNIOInfo const&, oceanbase::common::ObIOHandle&) at 0_cxx.cxx:?
oceanbase::common::ObIOManager::aio_write(oceanbase::common::ObSNIOInfo const&, oceanbase::common::ObIOHandle&) at ??:?
oceanbase::common::ObBackupIoAdapter::io_manager_write(char const*, long, long, oceanbase::common::ObIOFd const&, long&) at ??:?
oceanbase::common::ObBackupIoAdapter::write_single_file(oceanbase::common::ObString const&, oceanbase::share::ObBackupStorageInfo const*, char const*, long, oceanbase::common::ObStorageIdMod const&) at ??:?
oceanbase::archive::ObArchiveSender::handle(oceanbase::archive::ObArchiveSendTask&, oceanbase::archive::ObArchiveSender::TaskConsumeStatus&) at 0_cxx.cxx:?
oceanbase::archive::ObArchiveSender::run1() at 0_cxx.cxx:?
oceanbase::lib::Threads::run(long) at ??:?
oceanbase::lib::__th_start(void*) at 0_cxx.cxx:?
?? ??:0

独善其身 · 2025 年6 月 5 日 10:13

都到内核源码级别了啊

sunnyli · 2025 年6 月 5 日 11:10

您好，看出问题了么，咱们方便给远程看下么

淇铭 · 2025 年6 月 5 日 11:23

我们先看一下看看是否有问题等会同步给你

sunnyli · 2025 年6 月 5 日 13:22

你好，有结果了么

淇铭 · 2025 年6 月 5 日 13:48

看了堆栈信息io占用不太多是符合预期的目前从堆栈信息看没有发现泄漏 grep MEMORY observer.log | grep IoControl 后面可以看IoControl的模块占用是否持续上升也可以看observer.log中是否有-4013 内存爆的问题

sunnyli · 2025 年6 月 5 日 14:40

observer.rar (2.3 MB)
还是存在-4013报错

淇铭 · 2025 年6 月 5 日 14:48

看你发了官方悬赏我们群里交流

AntTech_EQZRCL · 2025 年6 月 12 日 08:58

好复杂

oracle123 · 2025 年6 月 12 日 09:06

终于找到了类似的问题，感谢楼主的发帖，为排查问题提供了很大的帮助，不然真的不知道从哪里开始排查，太难了。

AntTech_LG · 2025 年6 月 12 日 09:16

感谢楼主

wxer · 2025 年6 月 12 日 09:18

我看看

淇铭 · 2025 年6 月 12 日 11:26

内存一直不释放，租户内存100% 参考这个悬赏帖是同一个用户提问