sh-4.2# obd display-trace 90431c06-4601-11ee-813c-0242ac130002

[2023-08-29 00:18:22.442] [DEBUG] - mkdir /root/.obd/lock/

[2023-08-29 00:18:22.446] [DEBUG] - unknown lock mode 

[2023-08-29 00:18:22.455] [DEBUG] - try to get share lock /root/.obd/lock/global

[2023-08-29 00:18:22.456] [DEBUG] - share lock `/root/.obd/lock/global`, count 1

[2023-08-29 00:18:22.456] [DEBUG] - cmd: ['obcluster']

[2023-08-29 00:18:22.457] [DEBUG] - opts: {'servers': None, 'components': None, 'force_delete': None, 'strict_check': None, 'without_parameter': None}

[2023-08-29 00:18:22.458] [DEBUG] - Get Deploy by name

[2023-08-29 00:18:22.459] [DEBUG] - mkdir /root/.obd/cluster/

[2023-08-29 00:18:22.462] [DEBUG] - mkdir /root/.obd/config_parser/

[2023-08-29 00:18:22.472] [DEBUG] - try to get exclusive lock /root/.obd/lock/deploy_obcluster

[2023-08-29 00:18:22.473] [DEBUG] - exclusive lock `/root/.obd/lock/deploy_obcluster`, count 1

[2023-08-29 00:18:22.494] [DEBUG] - Deploy status judge

[2023-08-29 00:18:22.496] [INFO] Get local repositories

[2023-08-29 00:18:22.497] [DEBUG] - mkdir /root/.obd/repository

[2023-08-29 00:18:22.502] [DEBUG] - Get local repository oceanbase-ce-4.1.0.0-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:22.505] [DEBUG] - Search repository oceanbase-ce version: 4.1.0.0, tag: 8439ecf8db5e0649bd49671b41ea9e8c85756b63, release: None, package_hash: None

[2023-08-29 00:18:22.515] [DEBUG] - try to get share lock /root/.obd/lock/mirror_and_repo

[2023-08-29 00:18:22.516] [DEBUG] - share lock `/root/.obd/lock/mirror_and_repo`, count 1

[2023-08-29 00:18:22.517] [DEBUG] - mkdir /root/.obd/repository/oceanbase-ce

[2023-08-29 00:18:22.533] [DEBUG] - Found repository oceanbase-ce-4.1.0.0-100000192023032010.el7-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:22.534] [DEBUG] - Get local repository obagent-1.3.0-d57fbb4962b2fbecb6282358c59295fdfba4d6ac

[2023-08-29 00:18:22.535] [DEBUG] - Search repository obagent version: 1.3.0, tag: d57fbb4962b2fbecb6282358c59295fdfba4d6ac, release: None, package_hash: None

[2023-08-29 00:18:22.536] [DEBUG] - share lock `/root/.obd/lock/mirror_and_repo`, count 2

[2023-08-29 00:18:22.537] [DEBUG] - mkdir /root/.obd/repository/obagent

[2023-08-29 00:18:22.553] [DEBUG] - Found repository obagent-1.3.0-22.el7-d57fbb4962b2fbecb6282358c59295fdfba4d6ac

[2023-08-29 00:18:22.554] [DEBUG] - Get deploy config

[2023-08-29 00:18:22.595] [INFO] Search plugins

[2023-08-29 00:18:22.596] [DEBUG] - Searching start_check plugin for components ...

[2023-08-29 00:18:22.597] [DEBUG] - Searching start_check plugin for oceanbase-ce-4.1.0.0-100000192023032010.el7-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:22.598] [DEBUG] - mkdir /root/.obd/plugins

[2023-08-29 00:18:22.627] [DEBUG] - Found for oceanbase-ce-py_script_start_check-4.0.0.0 for oceanbase-ce-4.1.0.0

[2023-08-29 00:18:22.629] [DEBUG] - Searching start_check plugin for obagent-1.3.0-22.el7-d57fbb4962b2fbecb6282358c59295fdfba4d6ac

[2023-08-29 00:18:22.653] [DEBUG] - Found for obagent-py_script_start_check-1.3.0 for obagent-1.3.0

[2023-08-29 00:18:22.655] [DEBUG] - Searching create_tenant plugin for components ...

[2023-08-29 00:18:22.656] [DEBUG] - Searching create_tenant plugin for oceanbase-ce-4.1.0.0-100000192023032010.el7-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:22.672] [DEBUG] - Found for oceanbase-ce-py_script_create_tenant-4.0.0.0 for oceanbase-ce-4.1.0.0

[2023-08-29 00:18:22.673] [DEBUG] - Searching create_tenant plugin for obagent-1.3.0-22.el7-d57fbb4962b2fbecb6282358c59295fdfba4d6ac

[2023-08-29 00:18:22.683] [DEBUG] - No such create_tenant plugin for obagent-1.3.0

[2023-08-29 00:18:22.684] [DEBUG] - Searching start plugin for components ...

[2023-08-29 00:18:22.684] [DEBUG] - Searching start plugin for oceanbase-ce-4.1.0.0-100000192023032010.el7-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:22.700] [DEBUG] - Found for oceanbase-ce-py_script_start-4.0.0.0 for oceanbase-ce-4.1.0.0

[2023-08-29 00:18:22.702] [DEBUG] - Searching start plugin for obagent-1.3.0-22.el7-d57fbb4962b2fbecb6282358c59295fdfba4d6ac

[2023-08-29 00:18:22.712] [DEBUG] - Found for obagent-py_script_start-1.3.0 for obagent-1.3.0

[2023-08-29 00:18:22.713] [DEBUG] - Searching connect plugin for components ...

[2023-08-29 00:18:22.714] [DEBUG] - Searching connect plugin for oceanbase-ce-4.1.0.0-100000192023032010.el7-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:22.729] [DEBUG] - Found for oceanbase-ce-py_script_connect-3.1.0 for oceanbase-ce-4.1.0.0

[2023-08-29 00:18:22.730] [DEBUG] - Searching connect plugin for obagent-1.3.0-22.el7-d57fbb4962b2fbecb6282358c59295fdfba4d6ac

[2023-08-29 00:18:22.741] [DEBUG] - Found for obagent-py_script_connect-1.3.0 for obagent-1.3.0

[2023-08-29 00:18:22.742] [DEBUG] - Searching bootstrap plugin for components ...

[2023-08-29 00:18:22.742] [DEBUG] - Searching bootstrap plugin for oceanbase-ce-4.1.0.0-100000192023032010.el7-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:22.756] [DEBUG] - Found for oceanbase-ce-py_script_bootstrap-4.0.0.0 for oceanbase-ce-4.1.0.0

[2023-08-29 00:18:22.757] [DEBUG] - Searching bootstrap plugin for obagent-1.3.0-22.el7-d57fbb4962b2fbecb6282358c59295fdfba4d6ac

[2023-08-29 00:18:22.769] [DEBUG] - Found for obagent-py_script_bootstrap-0.1 for obagent-1.3.0

[2023-08-29 00:18:22.770] [DEBUG] - Searching display plugin for components ...

[2023-08-29 00:18:22.771] [DEBUG] - Searching display plugin for oceanbase-ce-4.1.0.0-100000192023032010.el7-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:22.786] [DEBUG] - Found for oceanbase-ce-py_script_display-3.1.0 for oceanbase-ce-4.1.0.0

[2023-08-29 00:18:22.787] [DEBUG] - Searching display plugin for obagent-1.3.0-22.el7-d57fbb4962b2fbecb6282358c59295fdfba4d6ac

[2023-08-29 00:18:22.797] [DEBUG] - Found for obagent-py_script_display-1.3.0 for obagent-1.3.0

[2023-08-29 00:18:22.799] [INFO] Open ssh connection

[2023-08-29 00:18:22.802] [INFO] Load cluster param plugin

[2023-08-29 00:18:22.804] [DEBUG] - Searching param plugin for components ...

[2023-08-29 00:18:22.806] [DEBUG] - Search param plugin for oceanbase-ce

[2023-08-29 00:18:22.824] [DEBUG] - Found for oceanbase-ce-param-4.0.0.0 for oceanbase-ce-4.1.0.0

[2023-08-29 00:18:22.825] [DEBUG] - Applying oceanbase-ce-param-4.0.0.0 for oceanbase-ce-4.1.0.0-100000192023032010.el7-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:23.411] [DEBUG] - Search param plugin for obagent

[2023-08-29 00:18:23.425] [DEBUG] - Found for obagent-param-1.3.0 for obagent-1.3.0

[2023-08-29 00:18:23.426] [DEBUG] - Applying obagent-param-1.3.0 for obagent-1.3.0-22.el7-d57fbb4962b2fbecb6282358c59295fdfba4d6ac

[2023-08-29 00:18:23.502] [DEBUG] - Call oceanbase-ce-py_script_start_check-4.0.0.0 for oceanbase-ce-4.1.0.0-100000192023032010.el7-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:23.503] [DEBUG] - import start_check

[2023-08-29 00:18:23.555] [DEBUG] - add start_check ref count to 1

[2023-08-29 00:18:23.564] [INFO] Check before start observer

[2023-08-29 00:18:23.567] [DEBUG] -- local execute: ls /root/ob/store/clog/tenant_1/ 

[2023-08-29 00:18:23.588] [DEBUG] -- exited code 2, error output:

[2023-08-29 00:18:23.590] [DEBUG] ls: cannot access /root/ob/store/clog/tenant_1/: No such file or directory

[2023-08-29 00:18:23.594] [DEBUG] 

[2023-08-29 00:18:23.597] [DEBUG] -- local execute: cat /root/ob/run/observer.pid 

[2023-08-29 00:18:23.611] [DEBUG] -- exited code 1, error output:

[2023-08-29 00:18:23.613] [DEBUG] cat: /root/ob/run/observer.pid: No such file or directory

[2023-08-29 00:18:23.614] [DEBUG] 

[2023-08-29 00:18:23.615] [DEBUG] -- 127.0.0.1 port check

[2023-08-29 00:18:23.617] [DEBUG] -- local execute: bash -c 'cat /proc/net/{tcp*,udp*}' | awk -F' ' '{print $2,$10}' | grep '00000000:0B41' | awk -F' ' '{print $2}' | uniq 

[2023-08-29 00:18:23.638] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.640] [DEBUG] -- local execute: bash -c 'cat /proc/net/{tcp*,udp*}' | awk -F' ' '{print $2,$10}' | grep '00000000:0B42' | awk -F' ' '{print $2}' | uniq 

[2023-08-29 00:18:23.660] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.666] [DEBUG] -- local execute: ls /root/ob/store/sstable/block_file 

[2023-08-29 00:18:23.686] [DEBUG] -- exited code 2, error output:

[2023-08-29 00:18:23.688] [DEBUG] ls: cannot access /root/ob/store/sstable/block_file: No such file or directory

[2023-08-29 00:18:23.689] [DEBUG] 

[2023-08-29 00:18:23.690] [DEBUG] -- local execute: grep -e '^ *lo:' /proc/net/dev 

[2023-08-29 00:18:23.702] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.704] [DEBUG] -- local execute: cat /proc/sys/fs/aio-max-nr /proc/sys/fs/aio-nr 

[2023-08-29 00:18:23.717] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.720] [DEBUG] -- local execute: ulimit -a 

[2023-08-29 00:18:23.729] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.731] [DEBUG] -- local execute: cat /proc/meminfo 

[2023-08-29 00:18:23.743] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.746] [DEBUG] -- local execute: df --block-size=1024  

[2023-08-29 00:18:23.760] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.762] [DEBUG] -- get disk info for path /, total: 405507829760 avail: 381667192832

[2023-08-29 00:18:23.764] [DEBUG] -- get disk info for path /dev, total: 67108864 avail: 67108864

[2023-08-29 00:18:23.765] [DEBUG] -- get disk info for path /dev/shm, total: 67108864 avail: 67108864

[2023-08-29 00:18:23.766] [DEBUG] -- get disk info for path /root/ob, total: 47793781587968 avail: 4155472838656

[2023-08-29 00:18:23.767] [DEBUG] -- get disk info for path /etc/hosts, total: 405507829760 avail: 381667192832

[2023-08-29 00:18:23.768] [DEBUG] -- get disk info for path /proc/acpi, total: 16821067776 avail: 16821067776

[2023-08-29 00:18:23.769] [DEBUG] -- get disk info for path /sys/firmware, total: 16821067776 avail: 16821067776

[2023-08-29 00:18:23.770] [DEBUG] -- local execute: df --block-size=1024 /root/ob/store 

[2023-08-29 00:18:23.789] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.792] [DEBUG] -- get disk info for path /root/ob, total: 47793781587968 avail: 4155472838656

[2023-08-29 00:18:23.793] [DEBUG] -- local execute: df --block-size=1024 /root/ob/store/clog 

[2023-08-29 00:18:23.810] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.812] [DEBUG] -- get disk info for path /root/ob, total: 47793781587968 avail: 4155472838656

[2023-08-29 00:18:23.813] [DEBUG] -- disk: {'/': {'total': 405507829760, 'avail': 381667192832, 'need': 0}, '/dev': {'total': 67108864, 'avail': 67108864, 'need': 0}, '/dev/shm': {'total': 67108864, 'avail': 67108864, 'need': 0}, '/root/ob': {'total': 47793781587968, 'avail': 4155472838656, 'need': 0}, '/etc/hosts': {'total': 405507829760, 'avail': 381667192832, 'need': 0}, '/proc/acpi': {'total': 16821067776, 'avail': 16821067776, 'need': 0}, '/sys/firmware': {'total': 16821067776, 'avail': 16821067776, 'need': 0}}

[2023-08-29 00:18:23.814] [WARNING] OBD-1012: (127.0.0.1) clog and data use the same disk (/root/ob)

[2023-08-29 00:18:23.816] [DEBUG] -- local execute: date +%s%N 

[2023-08-29 00:18:23.828] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.830] [DEBUG] -- 127.0.0.1 time delta -3.4228515625

[2023-08-29 00:18:23.833] [INFO] [WARN] OBD-1012: (127.0.0.1) clog and data use the same disk (/root/ob)

[2023-08-29 00:18:23.834] [INFO] 

[2023-08-29 00:18:23.837] [DEBUG] - sub start_check ref count to 0

[2023-08-29 00:18:23.839] [DEBUG] - export start_check

[2023-08-29 00:18:23.841] [DEBUG] - Call obagent-py_script_start_check-1.3.0 for obagent-1.3.0-22.el7-d57fbb4962b2fbecb6282358c59295fdfba4d6ac

[2023-08-29 00:18:23.842] [DEBUG] - import start_check

[2023-08-29 00:18:23.870] [DEBUG] - add start_check ref count to 1

[2023-08-29 00:18:23.876] [INFO] Check before start obagent

[2023-08-29 00:18:23.881] [DEBUG] -- local execute: cat /root/obagent/run/ob_agentd.pid 

[2023-08-29 00:18:23.889] [DEBUG] -- exited code 1, error output:

[2023-08-29 00:18:23.891] [DEBUG] cat: /root/obagent/run/ob_agentd.pid: No such file or directory

[2023-08-29 00:18:23.892] [DEBUG] 

[2023-08-29 00:18:23.893] [DEBUG] -- 127.0.0.1 port check

[2023-08-29 00:18:23.894] [DEBUG] -- local execute: bash -c 'cat /proc/net/{tcp*,udp*}' | awk -F' ' '{print $2,$10}' | grep '00000000:1F99' | awk -F' ' '{print $2}' | uniq 

[2023-08-29 00:18:23.914] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.916] [DEBUG] -- local execute: bash -c 'cat /proc/net/{tcp*,udp*}' | awk -F' ' '{print $2,$10}' | grep '00000000:1F98' | awk -F' ' '{print $2}' | uniq 

[2023-08-29 00:18:23.930] [DEBUG] -- exited code 0

[2023-08-29 00:18:23.932] [DEBUG] - sub start_check ref count to 0

[2023-08-29 00:18:23.933] [DEBUG] - export start_check

[2023-08-29 00:18:23.934] [DEBUG] - Call oceanbase-ce-py_script_start-4.0.0.0 for oceanbase-ce-4.1.0.0-100000192023032010.el7-8439ecf8db5e0649bd49671b41ea9e8c85756b63

[2023-08-29 00:18:23.934] [DEBUG] - import start

[2023-08-29 00:18:23.949] [DEBUG] - add start ref count to 1

[2023-08-29 00:18:23.952] [INFO] Start observer

[2023-08-29 00:18:23.953] [DEBUG] -- local execute: ls /root/ob/store/clog/tenant_1/ 

[2023-08-29 00:18:23.964] [DEBUG] -- exited code 2, error output:

[2023-08-29 00:18:23.966] [DEBUG] ls: cannot access /root/ob/store/clog/tenant_1/: No such file or directory

[2023-08-29 00:18:23.966] [DEBUG] 

[2023-08-29 00:18:23.967] [DEBUG] -- local execute: cat /root/ob/run/observer.pid 

[2023-08-29 00:18:23.978] [DEBUG] -- exited code 1, error output:

[2023-08-29 00:18:23.979] [DEBUG] cat: /root/ob/run/observer.pid: No such file or directory

[2023-08-29 00:18:23.979] [DEBUG] 

[2023-08-29 00:18:23.980] [DEBUG] -- 127.0.0.1 start command construction

[2023-08-29 00:18:23.981] [DEBUG] -- starting 127.0.0.1 observer

[2023-08-29 00:18:23.983] [DEBUG] -- root@127.0.0.1 set env LD_LIBRARY_PATH to '/root/ob/lib:'

[2023-08-29 00:18:23.985] [DEBUG] -- local execute: cd /root/ob; /root/ob/bin/observer -r '127.0.0.1:2882:2881' -p 2881 -P 2882 -z 'zone1' -n 'obcluster' -c 1 -d '/root/ob/store' -i 'lo' -l 'INFO' -o __min_full_resource_pool_memory=2147483648,memory_limit='6G',system_memory='1G',datafile_size='5G',log_disk_size='15G',cpu_count=16,enable_syslog_wf=False,enable_syslog_recycle=True,max_syslog_file_count=4 

[2023-08-29 00:18:25.088] [DEBUG] -- exited code 0

[2023-08-29 00:18:25.089] [DEBUG] -- root@127.0.0.1 set env LD_LIBRARY_PATH to '/tmp/_MEICZUVGM'

[2023-08-29 00:18:25.096] [INFO] observer program health check

[2023-08-29 00:18:28.100] [DEBUG] -- 127.0.0.1 program health check

[2023-08-29 00:18:28.103] [DEBUG] -- local execute: cat /root/ob/run/observer.pid 

[2023-08-29 00:18:28.117] [DEBUG] -- exited code 1, error output:

[2023-08-29 00:18:28.119] [DEBUG] cat: /root/ob/run/observer.pid: Operation not permitted

[2023-08-29 00:18:28.120] [DEBUG] 

[2023-08-29 00:18:28.122] [WARNING] OBD-2002: Failed to start 127.0.0.1 observer

[2023-08-29 00:18:28.124] [DEBUG] - sub start ref count to 0

[2023-08-29 00:18:28.126] [DEBUG] - export start

[2023-08-29 00:18:28.128] [ERROR] oceanbase-ce start failed

[2023-08-29 00:18:28.130] [INFO] See https://www.oceanbase.com/product/ob-deployer/error-codes .

[2023-08-29 00:18:28.132] [INFO] Trace ID: 90431c06-4601-11ee-813c-0242ac130002

[2023-08-29 00:18:28.134] [INFO] If you want to view detailed obd logs, please run: obd display-trace 90431c06-4601-11ee-813c-0242ac130002

[2023-08-29 00:18:28.136] [DEBUG] - share lock /root/.obd/lock/mirror_and_repo release, count 1

[2023-08-29 00:18:28.137] [DEBUG] - share lock /root/.obd/lock/mirror_and_repo release, count 0

[2023-08-29 00:18:28.137] [DEBUG] - unlock /root/.obd/lock/mirror_and_repo

[2023-08-29 00:18:28.138] [DEBUG] - exclusive lock /root/.obd/lock/deploy_obcluster release, count 0

[2023-08-29 00:18:28.139] [DEBUG] - unlock /root/.obd/lock/deploy_obcluster

[2023-08-29 00:18:28.140] [DEBUG] - share lock /root/.obd/lock/global release, count 0

[2023-08-29 00:18:28.141] [DEBUG] - unlock /root/.obd/lock/global