【 使用环境 】生产环境 or 测试环境
【 OB or 其他组件 】
【 使用版本 】
【问题描述】清晰明确描述问题
【复现路径】问题出现前后相关操作
【问题现象及影响】
############{EXECUTE}{2023-07-14T15:59:41.729+08:00}############
2023-07-14 15:59:41.732 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.c.m.t.model.SubtaskInstanceEntity : Run subtask, id=30636, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, primary_root_sys_password=******, task_instance_id=26645, task_operation=execute, cluster_type=STANDBY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, cluster_id=7, zone2.idc=JF1, sub_task_instance_id=30636, cluster_version=3.2.4.1, cluster_name=obcluster1, target_server_status=RUNNING, startup_option_string=cpu_count=16,datafile_disk_percentage=30,memory_limit=30GB, ob_version=3.2.4.1, service_name=obcluster1:1689076304, target_zone_status=RUNNING, zone3.idc=JF1, ob_cluster_id=1689076304, zone1.idc=JF1, service_type=OB_CLUSTER, rpm_name=oceanbase-3.2.4.1-101000052023010822.el7.x86_64.rpm, primary_rs_info=10.247.218.135:2882,10.247.62.51:2882,10.30.54.133:2882, target_cluster_status=RUNNING, latest_execution_start_time=2023-07-14T15:59:41.727+08:00, primary_ob_cluster_id=1689076302, ob_data_disk_path=/data/1}, listMap={zone3.host_ids=[10], root_servers=[8, 9, 10], server_ids=[17, 18, 19], dep_rpm_names=[], exists_server_addrs=[10.247.218.135:2882, 10.247.62.51:2882, 10.30.54.133:2882], zone2.host_ids=[9], host_ids=[8, 9, 10], zone1.host_ids=[8], ob_primary_connect_address_list=[10.247.218.135:2881, 10.247.62.51:2881, 10.30.54.133:2881], zone_names=[zone1, zone2, zone3]}}, executor=76.76.35.89
2023-07-14 15:59:41.763 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/start, request body:StartObServerProcessRequest(obPath=ObPath(installPath=/home/admin/oceanbase, dataPath=/data/1, logPath=/data/log1), withArguments=true, arguments=StartObServerProcessArguments(serverIp=10.30.63.23, zoneName=zone3, configUrl=http://10.50.157.90:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster1, rsList=10.30.51.181:2882:2881;10.30.52.133:2882:2881;10.30.63.23:2882:2881, clusterName=obcluster1, obClusterId=1689076304, optionString=cpu_count=16,datafile_disk_percentage=30,memory_limit=30GB), readOnly=false), params:null
2023-07-14 15:59:41.997 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/start, request body:StartObServerProcessRequest(obPath=ObPath(installPath=/home/admin/oceanbase, dataPath=/data/1, logPath=/data/log1), withArguments=true, arguments=StartObServerProcessArguments(serverIp=10.30.52.133, zoneName=zone2, configUrl=http://10.50.157.90:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster1, rsList=10.30.51.181:2882:2881;10.30.52.133:2882:2881;10.30.63.23:2882:2881, clusterName=obcluster1, obClusterId=1689076304, optionString=cpu_count=16,datafile_disk_percentage=30,memory_limit=30GB), readOnly=false), params:null
2023-07-14 15:59:42.221 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/start, request body:StartObServerProcessRequest(obPath=ObPath(installPath=/home/admin/oceanbase, dataPath=/data/1, logPath=/data/log1), withArguments=true, arguments=StartObServerProcessArguments(serverIp=10.30.51.181, zoneName=zone1, configUrl=http://10.50.157.90:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster1, rsList=10.30.51.181:2882:2881;10.30.52.133:2882:2881;10.30.63.23:2882:2881, clusterName=obcluster1, obClusterId=1689076304, optionString=cpu_count=16,datafile_disk_percentage=30,memory_limit=30GB), readOnly=false), params:null
2023-07-14 15:59:42.427 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : begin to check server accessible, hostIds=[8, 9, 10]
2023-07-14 15:59:42.438 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:42.509 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 15:59:42.519 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:42.612 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 15:59:42.623 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:42.718 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 15:59:42.719 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 15:59:42.720 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 15:59:47.731 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:47.807 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 15:59:47.819 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:47.915 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 15:59:47.933 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:48.038 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 15:59:48.039 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 15:59:48.040 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 15:59:53.051 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:53.126 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 15:59:53.138 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:53.233 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 15:59:53.244 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:53.346 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 15:59:53.348 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 15:59:53.348 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 15:59:58.360 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:58.434 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 15:59:58.445 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:58.541 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 15:59:58.551 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 15:59:58.655 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 15:59:58.656 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 15:59:58.657 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 16:00:03.668 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:03.744 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 16:00:03.754 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:03.850 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 16:00:03.861 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:03.961 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 16:00:03.962 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 16:00:03.963 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 16:00:08.975 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:09.050 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 16:00:09.063 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:09.158 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 16:00:09.169 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:09.274 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 16:00:09.275 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 16:00:09.276 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 16:00:14.287 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:14.363 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 16:00:14.375 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:14.469 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 16:00:14.480 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:14.583 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 16:00:14.584 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 16:00:14.585 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 16:00:19.597 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:19.672 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 16:00:19.687 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:19.783 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 16:00:19.793 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:19.896 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 16:00:19.897 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 16:00:19.898 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 16:00:24.909 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:24.984 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 16:00:24.994 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:25.090 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 16:00:25.101 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:25.210 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 16:00:25.212 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 16:00:25.213 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 16:00:30.224 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:30.300 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 16:00:30.311 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:30.407 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 16:00:30.421 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:30.524 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 16:00:30.526 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 16:00:30.527 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 16:00:35.539 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:35.615 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 16:00:35.627 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:35.721 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 16:00:35.732 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:35.838 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 16:00:35.839 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 16:00:35.839 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 16:00:40.850 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:40.925 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 16:00:40.936 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:41.030 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=9, exitCode=1, output=
2023-07-14 16:00:41.041 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.63.23:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:41.145 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=10, exitCode=1, output=
2023-07-14 16:00:41.147 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.s.t.b.c.helper.ObServerTaskHelper : there exists server(s) still not accessible, hostIds=8,9,10
2023-07-14 16:00:41.148 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds
2023-07-14 16:00:46.160 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.51.181:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:46.235 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] .a.o.s.o.o.c.ClusterHostOperationService : server not accessible, hostId=8, exitCode=1, output=
2023-07-14 16:00:46.246 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.30.52.133:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null
2023-07-14 16:00:58.292 INFO 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 30636, current state: RUNNING, new state: FAILED
2023-07-14 16:00:58.298 WARN 52 — [pool-manual-subtask-executor7,0001176647b94c2e,6fdcb9128892] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=30636, name=Bootstrap ob, state=FAILED, operation=EXECUTE, className=com.alipay.ocp.service.task.business.cluster.BootStrapObTask, seriesId=44, startTime=2023-07-14T15:59:41.727+08:00, endTime=2023-07-14T16:00:58.293+08:00}
java.lang.RuntimeException: wait all ob server accessible timeout
at com.alipay.ocp.service.task.business.cluster.helper.ObServerTaskHelper.waitAllObServerAccessible(ObServerTaskHelper.java:49) ~[ocp-service-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.service.task.business.cluster.BootStrapObTask.run(BootStrapObTask.java:144) ~[ocp-service-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.run(SubtaskInstanceEntity.java:225) ~[ocp-core-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:26) ~[ocp-core-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.core.task.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.3-20220905.jar!/:3.3.3-20220905]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312]
at java.util.concurrent.Thre
adPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312]
at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312]
Caused by: com.oceanbase.ocp.executor.exception.HttpRequestFailedException: [AgentClient]:http request is failed, response:AgentResponse(successful=false, timestamp=2023-07-14T16:00:58.273932234+08:00, duration=12006, status=Internal Server Error, traceId=0001176647b94c2e, server=10.30.52.133, data=null, error=code=1500,message=Execute shell command failed: error when execute shell command mysql -h127.1 -P2881 -uroot -e “select 1 from dual;”: Command timed out.,debugMessage=,subErrors=,)
at com.oceanbase.ocp.executor.internal.template.HttpTemplate.checkSuccess(HttpTemplate.java:391) ~[command-executor-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.oceanbase.ocp.executor.internal.template.HttpTemplate.doPost(HttpTemplate.java:248) ~[command-executor-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.oceanbase.ocp.executor.internal.template.HttpTemplate.post(HttpTemplate.java:101) ~[command-executor-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.oceanbase.ocp.executor.executor.AgentExecutor.accessObServerProcess(AgentExecutor.java:278) ~[command-executor-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.service.operation.ob.cluster.ClusterHostOperationService.isObServerProcessAccessible(ClusterHostOperationService.java:198) ~[ocp-service-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.service.task.business.cluster.helper.ObServerTaskHelper.isObServerAccessible(ObServerTaskHelper.java:94) ~[ocp-service-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.service.task.business.cluster.helper.ObServerTaskHelper.isAllObServerAccessible(ObServerTaskHelper.java:75) ~[ocp-service-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.service.task.business.cluster.helper.ObServerTaskHelper.lambda$waitAllObServerAccessible$0(ObServerTaskHelper.java:43) ~[ocp-service-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.common.pattern.Retry.executeUntilWithLimit(Retry.java:72
) ~[ocp-common-3.3.3-20220905.jar!/:3.3.3-20220905]
at com.alipay.ocp.service.task.business.cluster.helper.ObServerTaskHelper.waitAllObServerAccessible(ObServerTaskHelper.java:43) ~[ocp-service-3.3.3-20220905.jar!/:3.3.3-20220905]
… 12 common frames omitted
【附件】