【 使用环境 】测试环境
【 OB or 其他组件 】 V4.3.5
【问题描述】根据官网使用图形化界面安装oceanbase集群,安装完成之后随机一个observer节点的observer进程会异常退出,节点之间rpc端口通信没有问题,资源充足
请帮忙看看是哪里的问题
user:
username: root
password: letsg0
port: 22
oceanbase-ce:
version: 4.3.5.0
release: 100000202024123117.el7
package_hash: 48b61655aaa13e9b01b722928d1979c76b41937e
10.10.180.166:
zone: zone1
10.10.180.135:
zone: zone2
servers:
- 10.10.180.166
- 10.10.180.135
global:
appname: oceanbase
root_password: /,6Ib%}=yPf1aJh5k:*APaH97@
ocp_meta_password: CLjslrl0620@_
mysql_port: 2881
rpc_port: 2882
data_dir: /root/data
redo_dir: /root/redolog
home_path: /root/oceanbase/oceanbase
scenario: htap
cluster_id: 1738996760
ocp_agent_monitor_password: LBmXmFUspr
proxyro_password: n15WZsVOtH
enable_syslog_wf: true
max_syslog_file_count: 4
production_mode: false
memory_limit: 14G
datafile_size: 2G
system_memory: 6G
log_disk_size: 34G
cpu_count: 8
datafile_maxsize: 34G
datafile_next: 3G
obproxy-ce:
version: 4.3.2.0
package_hash: b7ae0af3860478f3caecaaec05bd67d0565e4021
release: 42.el7
servers:
- 10.10.180.26
global:
prometheus_listen_port: 2884
listen_port: 2883
rpc_listen_port: 2885
home_path: /root/oceanbase/obproxy
obproxy_sys_password: '*gIyJ@IatO0TlX[p2Jl'
skip_proxy_sys_private_check: true
enable_strict_kernel_release: false
enable_cluster_checkout: false
rs_list: 10.10.180.166:2881;10.10.180.135:2881
cluster_name: oceanbase
observer_root_password: /,6Ib%}=yPf1aJh5k:*APaH97@
10.10.180.26:
proxy_id: 4281
client_session_id_version: 2
depends:
- oceanbase-ce
obagent:
version: 4.2.2
package_hash: 19739a07a12eab736aff86ecf357b1ae660b554e
release: 100000042024011120.el7
servers:
- 10.10.180.166
- 10.10.180.135
global:
monagent_http_port: 8088
mgragent_http_port: 8089
home_path: /root/oceanbase/obagent
http_basic_auth_password: a1nOC6S9s
ob_monitor_status: active
depends:
- oceanbase-ce
ocp-express:
version: 4.2.2
package_hash: 09ffcf156d1df9318a78af52656f499d2315e3f7
release: 100000022024011120.el7
servers:
- 10.10.180.26
global:
port: 8180
admin_passwd: '{WAkHO75T4sa---Ou9'
home_path: /root/oceanbase/ocpexpress
ocp_root_password: r%05SbL_
memory_size: 812M
depends:
- obagent
- oceanbase-ce
- obproxy-ce
[2025-02-10 16:40:17.760] [DEBUG] - cmd: ['oceanbase']
[2025-02-10 16:40:17.760] [DEBUG] - opts: {}
[2025-02-10 16:40:17.760] [DEBUG] - mkdir /root/.obd/lock/
[2025-02-10 16:40:17.761] [DEBUG] - unknown lock mode
[2025-02-10 16:40:17.761] [DEBUG] - try to get share lock /root/.obd/lock/global
[2025-02-10 16:40:17.761] [DEBUG] - share lock `/root/.obd/lock/global`, count 1
[2025-02-10 16:40:17.761] [DEBUG] - Get Deploy by name
[2025-02-10 16:40:17.762] [DEBUG] - mkdir /root/.obd/cluster/
[2025-02-10 16:40:17.762] [DEBUG] - mkdir /root/.obd/config_parser/
[2025-02-10 16:40:17.762] [DEBUG] - try to get exclusive lock /root/.obd/lock/deploy_oceanbase
[2025-02-10 16:40:17.763] [DEBUG] - exclusive lock `/root/.obd/lock/deploy_oceanbase`, count 1
[2025-02-10 16:40:17.772] [DEBUG] - Deploy status judge
[2025-02-10 16:40:17.773] [DEBUG] - Get deploy config
[2025-02-10 16:40:17.813] [INFO] Get local repositories and plugins
[2025-02-10 16:40:17.814] [DEBUG] - mkdir /root/.obd/repository
[2025-02-10 16:40:17.815] [DEBUG] - Get local repository oceanbase-ce-4.3.5.0-48b61655aaa13e9b01b722928d1979c76b41937e
[2025-02-10 16:40:17.815] [DEBUG] - try to get share lock /root/.obd/lock/mirror_and_repo
[2025-02-10 16:40:17.815] [DEBUG] - share lock `/root/.obd/lock/mirror_and_repo`, count 1
[2025-02-10 16:40:17.819] [DEBUG] - Get local repository obproxy-ce-4.3.2.0-b7ae0af3860478f3caecaaec05bd67d0565e4021
[2025-02-10 16:40:17.819] [DEBUG] - share lock `/root/.obd/lock/mirror_and_repo`, count 2
[2025-02-10 16:40:17.822] [DEBUG] - Get local repository obagent-4.2.2-19739a07a12eab736aff86ecf357b1ae660b554e
[2025-02-10 16:40:17.822] [DEBUG] - share lock `/root/.obd/lock/mirror_and_repo`, count 3
[2025-02-10 16:40:17.825] [DEBUG] - Get local repository ocp-express-4.2.2-09ffcf156d1df9318a78af52656f499d2315e3f7
[2025-02-10 16:40:17.825] [DEBUG] - share lock `/root/.obd/lock/mirror_and_repo`, count 4
[2025-02-10 16:40:17.829] [DEBUG] - Searching param plugin for components ...
[2025-02-10 16:40:17.829] [DEBUG] - Search param plugin for oceanbase-ce
[2025-02-10 16:40:17.829] [DEBUG] - mkdir /root/.obd/plugins
[2025-02-10 16:40:17.830] [DEBUG] - Found for oceanbase-ce-param-4.3.3.0 for oceanbase-ce-4.3.5.0
[2025-02-10 16:40:17.830] [DEBUG] - Applying oceanbase-ce-param-4.3.3.0 for oceanbase-ce-4.3.5.0-100000202024123117.el7-48b61655aaa13e9b01b722928d1979c76b41937e
[2025-02-10 16:40:18.598] [DEBUG] - Search param plugin for obproxy-ce
[2025-02-10 16:40:18.599] [DEBUG] - Found for obproxy-ce-param-4.3.0 for obproxy-ce-4.3.2.0
[2025-02-10 16:40:18.599] [DEBUG] - Applying obproxy-ce-param-4.3.0 for obproxy-ce-4.3.2.0-42.el7-b7ae0af3860478f3caecaaec05bd67d0565e4021
[2025-02-10 16:40:18.791] [DEBUG] - Search param plugin for obagent
[2025-02-10 16:40:18.792] [DEBUG] - Found for obagent-param-4.2.2 for obagent-4.2.2
[2025-02-10 16:40:18.792] [DEBUG] - Applying obagent-param-4.2.2 for obagent-4.2.2-100000042024011120.el7-19739a07a12eab736aff86ecf357b1ae660b554e
[2025-02-10 16:40:18.885] [DEBUG] - Search param plugin for ocp-express
[2025-02-10 16:40:18.885] [DEBUG] - Found for ocp-express-param-4.1.0 for ocp-express-4.2.2
[2025-02-10 16:40:18.886] [DEBUG] - Applying ocp-express-param-4.1.0 for ocp-express-4.2.2-100000022024011120.el7-09ffcf156d1df9318a78af52656f499d2315e3f7
[2025-02-10 16:40:19.133] [INFO] Open ssh connection
[2025-02-10 16:40:19.140] [DEBUG] - host: 10.10.180.135, port: 22, user: root, password: ******
[2025-02-10 16:40:19.233] [DEBUG] - host: 10.10.180.166, port: 22, user: root, password: ******
[2025-02-10 16:40:19.323] [DEBUG] - host: 10.10.180.26, port: 22, user: root, password: ******
[2025-02-10 16:40:19.532] [DEBUG] - Searching display template for components ...
[2025-02-10 16:40:19.533] [DEBUG] - mkdir /root/.obd/workflows
[2025-02-10 16:40:19.535] [DEBUG] - Call workflow oceanbase-ce-py_script_workflow_display-0.1 for oceanbase-ce-4.3.5.0-100000202024123117.el7-48b61655aaa13e9b01b722928d1979c76b41937e
[2025-02-10 16:40:19.535] [DEBUG] - mkdir /root/.obd/mirror
[2025-02-10 16:40:19.535] [DEBUG] - mkdir /root/.obd/mirror/remote
[2025-02-10 16:40:19.535] [DEBUG] - mkdir /root/.obd/mirror/local
[2025-02-10 16:40:19.536] [DEBUG] - mkdir /root/.obd/optimize/
[2025-02-10 16:40:19.536] [DEBUG] - mkdir /root/.obd/tool/
[2025-02-10 16:40:19.536] [DEBUG] - import display
[2025-02-10 16:40:19.537] [DEBUG] - add display ref count to 1
[2025-02-10 16:40:19.538] [DEBUG] - sub display ref count to 0
[2025-02-10 16:40:19.538] [DEBUG] - export display
[2025-02-10 16:40:19.538] [DEBUG] - plugin oceanbase-ce-py_script_workflow_display-0.1 result: True
[2025-02-10 16:40:19.538] [DEBUG] - Found for oceanbase-ce-py_script_workflow_display-0.1 for oceanbase-ce-0.1
[2025-02-10 16:40:19.538] [DEBUG] - Searching display template for components ...
[2025-02-10 16:40:19.538] [DEBUG] - Call workflow obproxy-ce-py_script_workflow_display-0.1 for obproxy-ce-4.3.2.0-42.el7-b7ae0af3860478f3caecaaec05bd67d0565e4021
[2025-02-10 16:40:19.538] [DEBUG] - import display
[2025-02-10 16:40:19.539] [DEBUG] - add display ref count to 1
[2025-02-10 16:40:19.539] [DEBUG] - sub display ref count to 0
[2025-02-10 16:40:19.539] [DEBUG] - export display
[2025-02-10 16:40:19.539] [DEBUG] - plugin obproxy-ce-py_script_workflow_display-0.1 result: True
[2025-02-10 16:40:19.540] [DEBUG] - Found for obproxy-ce-py_script_workflow_display-0.1 for obproxy-ce-0.1
[2025-02-10 16:40:19.540] [DEBUG] - Searching display template for components ...
[2025-02-10 16:40:19.540] [DEBUG] - Call workflow obagent-py_script_workflow_display-0.1 for obagent-4.2.2-100000042024011120.el7-19739a07a12eab736aff86ecf357b1ae660b554e
[2025-02-10 16:40:19.540] [DEBUG] - import display
[2025-02-10 16:40:19.541] [DEBUG] - add display ref count to 1
[2025-02-10 16:40:19.541] [DEBUG] - sub display ref count to 0
[2025-02-10 16:40:19.541] [DEBUG] - export display
[2025-02-10 16:40:19.541] [DEBUG] - plugin obagent-py_script_workflow_display-0.1 result: True
[2025-02-10 16:40:19.541] [DEBUG] - Found for obagent-py_script_workflow_display-0.1 for obagent-0.1
[2025-02-10 16:40:19.541] [DEBUG] - Searching display template for components ...
[2025-02-10 16:40:19.542] [DEBUG] - Call workflow ocp-express-py_script_workflow_display-0.1 for ocp-express-4.2.2-100000022024011120.el7-09ffcf156d1df9318a78af52656f499d2315e3f7
[2025-02-10 16:40:19.542] [DEBUG] - import display
[2025-02-10 16:40:19.542] [DEBUG] - add display ref count to 1
[2025-02-10 16:40:19.543] [DEBUG] - sub display ref count to 0
[2025-02-10 16:40:19.543] [DEBUG] - export display
[2025-02-10 16:40:19.543] [DEBUG] - plugin ocp-express-py_script_workflow_display-0.1 result: True
[2025-02-10 16:40:19.543] [DEBUG] - Found for ocp-express-py_script_workflow_display-0.1 for ocp-express-0.1
[2025-02-10 16:40:19.543] [DEBUG] - share lock `/root/.obd/lock/mirror_and_repo`, count 5
[2025-02-10 16:40:19.547] [DEBUG] - Searching status plugin for components ...
[2025-02-10 16:40:19.547] [DEBUG] - Searching status plugin for oceanbase-ce-4.3.5.0-100000202024123117.el7-48b61655aaa13e9b01b722928d1979c76b41937e
[2025-02-10 16:40:19.548] [DEBUG] - Found for oceanbase-ce-py_script_status-3.1.0 for oceanbase-ce-4.3.5.0
[2025-02-10 16:40:19.548] [DEBUG] - Call plugin oceanbase-ce-py_script_status-3.1.0 for oceanbase-ce-4.3.5.0-100000202024123117.el7-48b61655aaa13e9b01b722928d1979c76b41937e
[2025-02-10 16:40:19.548] [DEBUG] - import status
[2025-02-10 16:40:19.549] [DEBUG] - add status ref count to 1
[2025-02-10 16:40:19.550] [DEBUG] -- root@10.10.180.166 execute: cat /root/oceanbase/oceanbase/run/observer.pid
[2025-02-10 16:40:19.596] [DEBUG] -- exited code 0
[2025-02-10 16:40:19.597] [DEBUG] -- root@10.10.180.166 execute: ls /proc/28469
[2025-02-10 16:40:19.681] [DEBUG] -- exited code 0
[2025-02-10 16:40:19.682] [DEBUG] -- root@10.10.180.135 execute: cat /root/oceanbase/oceanbase/run/observer.pid
[2025-02-10 16:40:19.725] [DEBUG] -- exited code 0
[2025-02-10 16:40:19.726] [DEBUG] -- root@10.10.180.135 execute: ls /proc/6570
[2025-02-10 16:40:19.806] [DEBUG] -- exited code 2, error output:
[2025-02-10 16:40:19.806] [DEBUG] ls: cannot access /proc/6570: No such file or directory
[2025-02-10 16:40:19.806] [DEBUG]
[2025-02-10 16:40:19.807] [DEBUG] - sub status ref count to 0
[2025-02-10 16:40:19.807] [DEBUG] - export status
[2025-02-10 16:40:19.807] [DEBUG] - plugin oceanbase-ce-py_script_status-3.1.0 result: True
[2025-02-10 16:40:19.808] [DEBUG] - Searching status_check plugin for components ...
[2025-02-10 16:40:19.808] [DEBUG] - Searching status_check plugin for general-4.3.5.0--None
[2025-02-10 16:40:19.809] [DEBUG] - Found for general-py_script_status_check-0.1 for general-4.3.5.0
[2025-02-10 16:40:19.809] [DEBUG] - Call plugin general-py_script_status_check-0.1 for oceanbase-ce-4.3.5.0-100000202024123117.el7-48b61655aaa13e9b01b722928d1979c76b41937e
[2025-02-10 16:40:19.809] [DEBUG] - import status_check
[2025-02-10 16:40:19.810] [DEBUG] - add status_check ref count to 1
[2025-02-10 16:40:19.811] [WARNING] 10.10.180.135 oceanbase-ce is not running
[2025-02-10 16:40:19.811] [DEBUG] - sub status_check ref count to 0
[2025-02-10 16:40:19.811] [DEBUG] - export status_check
[2025-02-10 16:40:19.811] [DEBUG] - plugin general-py_script_status_check-0.1 result: False
[2025-02-10 16:40:19.811] [DEBUG] - share lock /root/.obd/lock/mirror_and_repo release, count 4
[2025-02-10 16:40:19.812] [DEBUG] - share lock /root/.obd/lock/mirror_and_repo release, count 3
[2025-02-10 16:40:19.812] [DEBUG] - share lock /root/.obd/lock/mirror_and_repo release, count 2
[2025-02-10 16:40:19.812] [DEBUG] - share lock /root/.obd/lock/mirror_and_repo release, count 1
[2025-02-10 16:40:19.812] [DEBUG] - share lock /root/.obd/lock/mirror_and_repo release, count 0
[2025-02-10 16:40:19.812] [DEBUG] - unlock /root/.obd/lock/mirror_and_repo
[2025-02-10 16:40:19.812] [DEBUG] - exclusive lock /root/.obd/lock/deploy_oceanbase release, count 0
[2025-02-10 16:40:19.812] [DEBUG] - unlock /root/.obd/lock/deploy_oceanbase
[2025-02-10 16:40:19.812] [DEBUG] - share lock /root/.obd/lock/global release, count 0
[2025-02-10 16:40:19.812] [DEBUG] - unlock /root/.obd/lock/global
[2025-02-10 16:40:19.813] [INFO] See https://www.oceanbase.com/product/ob-deployer/error-codes .
[2025-02-10 16:40:19.813] [INFO] Trace ID: a7bee08c-e78a-11ef-9ff5-fa410d32c000
[2025-02-10 16:40:19.813] [INFO] If you want to view detailed obd logs, please run: obd display-trace a7bee08c-e78a-11ef-9ff5-fa410d32c000