【 使用环境 】测试环境
【 OB or 其他组件 】obd
【 使用版本 】 oceanbase-all-in-one-4.2.2.0-100000192024011915.el7.x86_64.tar.gz
【问题描述】
去年还能正常启动的数据库，很久没使用了，突然起不起来了。具体情况如下，请问下该怎么解决。
vim ~/.oceanbase-all-in-one/obd/usr/obd/example/mini-local-example.yaml
oceanbase-ce:
servers:
# Please don't use hostname, only IP can be supported
- 127.0.0.1
global:
# The working directory for OceanBase Database. OceanBase Database is started under this directory. This is a required field.
home_path: /root/observer
# The directory for data storage. The default value is $home_path/store.
data_dir: /DATA/oceanbaseData
# The directory for clog, ilog, and slog. The default value is the same as the data_dir value.
redo_dir: /DATA/oceanbaseLog
# Starting from observer version 4.2, the network selection for the observer is based on the 'local_ip' parameter, and the 'devname' parameter is no longer mandatory.
# If the 'local_ip' parameter is set, the observer will first use this parameter for the configuration, regardless of the 'devname' parameter.
# If only the 'devname' parameter is set, the observer will use the 'devname' parameter for the configuration.
# If neither the 'devname' nor the 'local_ip' parameters are set, the 'local_ip' parameter will be automatically assigned the IP address configured above.
# devname: eth0
mysql_port: 2881 # External port for OceanBase Database. The default value is 2881. DO NOT change this value after the cluster is started.
rpc_port: 2882 # Internal port for OceanBase Database. The default value is 2882. DO NOT change this value after the cluster is started.
zone: zone1
cluster_id: 1
# please set memory limit to a suitable value which is matching resource.
memory_limit: 8G # The maximum running memory for an observer
system_memory: 3G # The reserved system memory. system_memory is reserved for general tenants. The default value is 30G.
datafile_size: 2G # Size of the data file.
datafile_next: 2G # the auto extend step. Please enter an capacity, such as 2G
datafile_maxsize: 20G # the auto extend max size. Please enter an capacity, such as 20G
log_disk_size: 13G # The size of disk space used by the clog files.
cpu_count: 2
production_mode: false
enable_syslog_wf: false # Print system logs whose levels are higher than WARNING to a separate log file. The default value is true.
enable_syslog_recycle: true # Enable auto system log recycling or not. The default value is false.
max_syslog_file_count: 4 # The maximum number of reserved log files before enabling auto recycling. The default value is 0.
root_password: xx # root user password, can be empty
vim ~/.obd/cluster/obtest/config.yaml
oceanbase-ce:
servers:
# Please don't use hostname, only IP can be supported
- 127.0.0.1
global:
# The working directory for OceanBase Database. OceanBase Database is started under this directory. This is a required field.
home_path: /root/observer
# The directory for data storage. The default value is $home_path/store.
data_dir: /DATA/oceanbaseData
# The directory for clog, ilog, and slog. The default value is the same as the data_dir value.
redo_dir: /DATA/oceanbaseLog
# Starting from observer version 4.2, the network selection for the observer is based on the 'local_ip' parameter, and the 'devname' parameter is no longer mandatory.
# If the 'local_ip' parameter is set, the observer will first use this parameter for the configuration, regardless of the 'devname' parameter.
# If only the 'devname' parameter is set, the observer will use the 'devname' parameter for the configuration.
# If neither the 'devname' nor the 'local_ip' parameters are set, the 'local_ip' parameter will be automatically assigned the IP address configured above.
# devname: eth0
mysql_port: 2881 # External port for OceanBase Database. The default value is 2881. DO NOT change this value after the cluster is started.
rpc_port: 2882 # Internal port for OceanBase Database. The default value is 2882. DO NOT change this value after the cluster is started.
zone: zone1
cluster_id: 1
# please set memory limit to a suitable value which is matching resource.
memory_limit: 8G # The maximum running memory for an observer
system_memory: 3G # The reserved system memory. system_memory is reserved for general tenants. The default value is 30G.
datafile_size: 2G # Size of the data file.
datafile_next: 2G # the auto extend step. Please enter an capacity, such as 2G
datafile_maxsize: 20G # the auto extend max size. Please enter an capacity, such as 20G
log_disk_size: 13G # The size of disk space used by the clog files.
cpu_count: 2
production_mode: false
enable_syslog_wf: false # Print system logs whose levels are higher than WARNING to a separate log file. The default value is true.
enable_syslog_recycle: true # Enable auto system log recycling or not. The default value is false.
max_syslog_file_count: 4 # The maximum number of reserved log files before enabling auto recycling. The default value is 0.
root_password: xx # root user password, can be empty
obd display-trace e16efd9a-7352-11f0-9d45-001c297b7098
[2025-08-08 13:32:26.162] [DEBUG] - cmd: ['obtest']
[2025-08-08 13:32:26.163] [DEBUG] - opts: {'servers': None, 'components': None, 'force_delete': None, 'strict_check': None, 'without_parameter': None}
[2025-08-08 13:32:26.163] [DEBUG] - mkdir /root/.obd/lock/
[2025-08-08 13:32:26.163] [DEBUG] - unknown lock mode
[2025-08-08 13:32:26.163] [DEBUG] - try to get share lock /root/.obd/lock/global
[2025-08-08 13:32:26.164] [DEBUG] - share lock `/root/.obd/lock/global`, count 1
[2025-08-08 13:32:26.164] [DEBUG] - Get Deploy by name
[2025-08-08 13:32:26.164] [DEBUG] - mkdir /root/.obd/cluster/
[2025-08-08 13:32:26.164] [DEBUG] - mkdir /root/.obd/config_parser/
[2025-08-08 13:32:26.164] [DEBUG] - try to get exclusive lock /root/.obd/lock/deploy_obtest
[2025-08-08 13:32:26.164] [DEBUG] - exclusive lock `/root/.obd/lock/deploy_obtest`, count 1
[2025-08-08 13:32:26.170] [DEBUG] - Deploy status judge
[2025-08-08 13:32:26.170] [INFO] Get local repositories
[2025-08-08 13:32:26.171] [DEBUG] - mkdir /root/.obd/repository
[2025-08-08 13:32:26.172] [DEBUG] - Get local repository oceanbase-ce-4.2.2.0-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:26.172] [DEBUG] - Search repository oceanbase-ce version: 4.2.2.0, tag: 3771d4be8846700bf93e40f5965ffb349147ec21, release: None, package_hash: None
[2025-08-08 13:32:26.172] [DEBUG] - try to get share lock /root/.obd/lock/mirror_and_repo
[2025-08-08 13:32:26.172] [DEBUG] - share lock `/root/.obd/lock/mirror_and_repo`, count 1
[2025-08-08 13:32:26.172] [DEBUG] - mkdir /root/.obd/repository/oceanbase-ce
[2025-08-08 13:32:26.175] [DEBUG] - Found repository oceanbase-ce-4.2.2.0-100000192024011915.el8-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:26.306] [DEBUG] - Get deploy config
[2025-08-08 13:32:26.330] [INFO] Search plugins
[2025-08-08 13:32:26.331] [DEBUG] - Searching start_check plugin for components ...
[2025-08-08 13:32:26.331] [DEBUG] - Searching start_check plugin for oceanbase-ce-4.2.2.0-100000192024011915.el8-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:26.331] [DEBUG] - mkdir /root/.obd/plugins
[2025-08-08 13:32:26.333] [DEBUG] - Found for oceanbase-ce-py_script_start_check-4.2.2.0 for oceanbase-ce-4.2.2.0
[2025-08-08 13:32:26.333] [DEBUG] - Searching create_tenant plugin for components ...
[2025-08-08 13:32:26.333] [DEBUG] - Searching create_tenant plugin for oceanbase-ce-4.2.2.0-100000192024011915.el8-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:26.334] [DEBUG] - Found for oceanbase-ce-py_script_create_tenant-4.2.0.0 for oceanbase-ce-4.2.2.0
[2025-08-08 13:32:26.335] [DEBUG] - Searching start plugin for components ...
[2025-08-08 13:32:26.335] [DEBUG] - Searching start plugin for oceanbase-ce-4.2.2.0-100000192024011915.el8-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:26.337] [DEBUG] - Found for oceanbase-ce-py_script_start-4.2.2.0 for oceanbase-ce-4.2.2.0
[2025-08-08 13:32:26.337] [DEBUG] - Searching connect plugin for components ...
[2025-08-08 13:32:26.338] [DEBUG] - Searching connect plugin for oceanbase-ce-4.2.2.0-100000192024011915.el8-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:26.339] [DEBUG] - Found for oceanbase-ce-py_script_connect-4.2.2.0 for oceanbase-ce-4.2.2.0
[2025-08-08 13:32:26.339] [DEBUG] - Searching bootstrap plugin for components ...
[2025-08-08 13:32:26.340] [DEBUG] - Searching bootstrap plugin for oceanbase-ce-4.2.2.0-100000192024011915.el8-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:26.341] [DEBUG] - Found for oceanbase-ce-py_script_bootstrap-4.2.2.0 for oceanbase-ce-4.2.2.0
[2025-08-08 13:32:26.342] [DEBUG] - Searching display plugin for components ...
[2025-08-08 13:32:26.342] [DEBUG] - Searching display plugin for oceanbase-ce-4.2.2.0-100000192024011915.el8-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:26.343] [DEBUG] - Found for oceanbase-ce-py_script_display-3.1.0 for oceanbase-ce-4.2.2.0
[2025-08-08 13:32:26.465] [INFO] Load cluster param plugin
[2025-08-08 13:32:26.467] [DEBUG] - Get local repository oceanbase-ce-4.2.2.0-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:26.467] [DEBUG] - Searching param plugin for components ...
[2025-08-08 13:32:26.467] [DEBUG] - Search param plugin for oceanbase-ce
[2025-08-08 13:32:26.468] [DEBUG] - Found for oceanbase-ce-param-4.2.2.0 for oceanbase-ce-4.2.2.0
[2025-08-08 13:32:26.469] [DEBUG] - Applying oceanbase-ce-param-4.2.2.0 for oceanbase-ce-4.2.2.0-100000192024011915.el8-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:27.216] [INFO] Open ssh connection
[2025-08-08 13:32:27.351] [DEBUG] - Call oceanbase-ce-py_script_start_check-4.2.2.0 for oceanbase-ce-4.2.2.0-100000192024011915.el8-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:27.351] [DEBUG] - import start_check
[2025-08-08 13:32:27.370] [DEBUG] - add start_check ref count to 1
[2025-08-08 13:32:27.370] [INFO] Check before start observer
[2025-08-08 13:32:27.375] [DEBUG] -- local execute: ls /DATA/oceanbaseData/clog/tenant_1/
[2025-08-08 13:32:27.384] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.384] [DEBUG] -- local execute: cat /root/observer/run/observer.pid
[2025-08-08 13:32:27.390] [DEBUG] -- exited code 1, error output:
[2025-08-08 13:32:27.390] [DEBUG] cat: /root/observer/run/observer.pid: No such file or directory
[2025-08-08 13:32:27.390] [DEBUG]
[2025-08-08 13:32:27.390] [DEBUG] -- 127.0.0.1 port check
[2025-08-08 13:32:27.391] [DEBUG] -- local execute: bash -c 'cat /proc/net/{tcp*,udp*}' | awk -F' ' '{print $2,$10}' | grep '00000000:0B41' | awk -F' ' '{print $2}' | uniq
[2025-08-08 13:32:27.403] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.403] [DEBUG] -- local execute: bash -c 'cat /proc/net/{tcp*,udp*}' | awk -F' ' '{print $2,$10}' | grep '00000000:0B42' | awk -F' ' '{print $2}' | uniq
[2025-08-08 13:32:27.413] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.413] [DEBUG] -- local execute: bash -c 'cat /proc/net/{tcp*,udp*}' | awk -F' ' '{print $2,$10}' | grep '00000000:0B46' | awk -F' ' '{print $2}' | uniq
[2025-08-08 13:32:27.423] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.424] [DEBUG] -- local execute: ls /DATA/oceanbaseData/sstable/block_file
[2025-08-08 13:32:27.432] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.432] [DEBUG] -- local execute: [ -w /tmp/ ] || [ -w /tmp/obshell ]
[2025-08-08 13:32:27.436] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.436] [DEBUG] -- local execute: cat /proc/sys/fs/aio-max-nr /proc/sys/fs/aio-nr
[2025-08-08 13:32:27.440] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.440] [DEBUG] -- local execute: ulimit -a
[2025-08-08 13:32:27.444] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.445] [DEBUG] -- local execute: sysctl -a
[2025-08-08 13:32:27.463] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.468] [DEBUG] -- local execute: cat /proc/meminfo
[2025-08-08 13:32:27.475] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.476] [DEBUG] -- local execute: df --block-size=1024
[2025-08-08 13:32:27.481] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.481] [DEBUG] -- get disk info for path /dev, total: 4194304 avail: 4194304
[2025-08-08 13:32:27.481] [DEBUG] -- get disk info for path /dev/shm, total: 8119631872 avail: 8119631872
[2025-08-08 13:32:27.481] [DEBUG] -- get disk info for path /run, total: 3247853568 avail: 3238604800
[2025-08-08 13:32:27.481] [DEBUG] -- get disk info for path /sys/fs/cgroup, total: 4194304 avail: 4194304
[2025-08-08 13:32:27.482] [DEBUG] -- get disk info for path /, total: 25110331392 avail: 10587439104
[2025-08-08 13:32:27.482] [DEBUG] -- get disk info for path /tmp, total: 8119631872 avail: 8034779136
[2025-08-08 13:32:27.482] [DEBUG] -- get disk info for path /boot, total: 1020702720 avail: 608976896
[2025-08-08 13:32:27.482] [DEBUG] -- get disk info for path /DATA, total: 129792819200 avail: 34400968704
[2025-08-08 13:32:27.482] [DEBUG] -- disk: {'/dev': {'total': 4194304, 'avail': 4194304, 'need': 0}, '/dev/shm': {'total': 8119631872, 'avail': 8119631872, 'need': 0}, '/run': {'total': 3247853568, 'avail': 3238604800, 'need': 0}, '/sys/fs/cgroup': {'total': 4194304, 'avail': 4194304, 'need': 0}, '/': {'total': 25110331392, 'avail': 10587439104, 'need': 0}, '/tmp': {'total': 8119631872, 'avail': 8034779136, 'need': 0}, '/boot': {'total': 1020702720, 'avail': 608976896, 'need': 0}, '/DATA': {'total': 129792819200, 'avail': 34400968704, 'need': 0}}
[2025-08-08 13:32:27.482] [DEBUG] -- local execute: date +%s%N
[2025-08-08 13:32:27.487] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.488] [DEBUG] -- 127.0.0.1 time delta -1.75390625
[2025-08-08 13:32:27.510] [DEBUG] - sub start_check ref count to 0
[2025-08-08 13:32:27.510] [DEBUG] - export start_check
[2025-08-08 13:32:27.510] [DEBUG] - Call oceanbase-ce-py_script_start-4.2.2.0 for oceanbase-ce-4.2.2.0-100000192024011915.el8-3771d4be8846700bf93e40f5965ffb349147ec21
[2025-08-08 13:32:27.511] [DEBUG] - import start
[2025-08-08 13:32:27.516] [DEBUG] - add start ref count to 1
[2025-08-08 13:32:27.517] [INFO] Start observer
[2025-08-08 13:32:27.519] [DEBUG] -- local execute: ls /DATA/oceanbaseData/clog/tenant_1/
[2025-08-08 13:32:27.524] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.525] [DEBUG] -- local execute: cat /root/observer/run/observer.pid
[2025-08-08 13:32:27.535] [DEBUG] -- exited code 1, error output:
[2025-08-08 13:32:27.535] [DEBUG] cat: /root/observer/run/observer.pid: No such file or directory
[2025-08-08 13:32:27.535] [DEBUG]
[2025-08-08 13:32:27.535] [DEBUG] -- 127.0.0.1 start command construction
[2025-08-08 13:32:27.536] [DEBUG] -- starting 127.0.0.1 observer
[2025-08-08 13:32:27.537] [DEBUG] -- root@127.0.0.1 set env LD_LIBRARY_PATH to '/root/observer/lib:'
[2025-08-08 13:32:27.537] [DEBUG] -- local execute: cd /root/observer; /root/observer/bin/observer -r '127.0.0.1:2882:2881' -p 2881 -P 2882 -z 'zone1' -c 1 -d '/DATA/oceanbaseData' -I '127.0.0.1' -o __min_full_resource_pool_memory=2147483648,memory_limit='8G',system_memory='3G',datafile_size='2G',datafile_next='2G',datafile_maxsize='20G',log_disk_size='13G',cpu_count=2,enable_syslog_wf=False,enable_syslog_recycle=True,max_syslog_file_count=4
[2025-08-08 13:32:27.763] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.764] [DEBUG] -- root@127.0.0.1 delete env LD_LIBRARY_PATH
[2025-08-08 13:32:27.781] [DEBUG] -- start_obshell: True
[2025-08-08 13:32:27.782] [DEBUG] -- local execute: cat /root/observer/run/obshell.pid
[2025-08-08 13:32:27.791] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.792] [DEBUG] -- local execute: ls /proc/3124
[2025-08-08 13:32:27.799] [DEBUG] -- exited code 0
[2025-08-08 13:32:27.800] [DEBUG] -- 127.0.0.1 obshell[pid: 3124] started
[2025-08-08 13:32:27.801] [INFO] observer program health check
[2025-08-08 13:32:30.805] [DEBUG] -- 127.0.0.1 program health check
[2025-08-08 13:32:30.805] [DEBUG] -- local execute: cat /root/observer/run/observer.pid
[2025-08-08 13:32:30.815] [DEBUG] -- exited code 1, error output:
[2025-08-08 13:32:30.816] [DEBUG] cat: /root/observer/run/observer.pid: No such file or directory
[2025-08-08 13:32:30.816] [DEBUG]
[2025-08-08 13:32:30.941] [WARNING] OBD-2002: Failed to start 127.0.0.1 observer
[2025-08-08 13:32:30.942] [DEBUG] - sub start ref count to 0
[2025-08-08 13:32:30.942] [DEBUG] - export start
[2025-08-08 13:32:30.942] [ERROR] oceanbase-ce start failed
[2025-08-08 13:32:30.949] [INFO] See
[2025-08-08 13:32:30.949] [INFO] Trace ID: 114b180c-7419-11f0-8cb5-001c297b7098
[2025-08-08 13:32:30.950] [INFO] If you want to view detailed obd logs, please run: obd display-trace 114b180c-7419-11f0-8cb5-001c297b7098
[2025-08-08 13:32:30.950] [DEBUG] - share lock /root/.obd/lock/mirror_and_repo release, count 0
[2025-08-08 13:32:30.950] [DEBUG] - unlock /root/.obd/lock/mirror_and_repo
[2025-08-08 13:32:30.951] [DEBUG] - exclusive lock /root/.obd/lock/deploy_obtest release, count 0
[2025-08-08 13:32:30.951] [DEBUG] - unlock /root/.obd/lock/deploy_obtest
[2025-08-08 13:32:30.951] [DEBUG] - share lock /root/.obd/lock/global release, count 0
[2025-08-08 13:32:30.951] [DEBUG] - unlock /root/.obd/lock/global
【复现路径】
