############{EXECUTE}{2024-05-18T20:38:25.887+08:00}############ 2024-05-18 20:38:26.008 INFO 11 --- [pool-manual-subtask-executor6,b64e037e7e37490f,07e60bc8b48b] c.o.o.c.t.e.runner.JavaSubtaskRunner : Run subtask, id=3006037, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=3004886, task_operation=execute, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, link_obproxy=false, primary_zone=zone1;zone2;zone3, cluster_id=3000014, ob_svr_port=2882, zone2.idc=IDC2, sub_task_instance_id=3006037, cluster_version=3.2.4.4, cluster_name=obcluster1, target_server_status=RUNNING, mix_rpm_names={"zone3":"oceanbase-3.2.4.4-104050012023073116.el7.x86_64.rpm","zone2":"oceanbase-3.2.4.4-104050012023073116.el7.x86_64.rpm","zone1":"oceanbase-3.2.4.4-104050012023073116.el7.x86_64.rpm"}, startup_option_string=datafile_disk_percentage=50,system_memory=10G, ob_version=3.2.4.4, service_name=obcluster1:1715516302, target_zone_status=RUNNING, zone3.idc=IDC3, ob_cluster_id=1715516302, zone1.idc=IDC1, root_sys_password=******, service_type=OB_CLUSTER, obproxy_proxyro_password=******, target_cluster_status=RUNNING, latest_execution_start_time=2024-05-18T20:38:25.879+08:00, ob_data_disk_path=/data/1}, listMap={zone3.host_ids=[3000003], root_servers=[3000001, 3000002, 3000003], server_ids=[3000040, 3000041, 3000042], dep_rpm_names=[], zone2.host_ids=[3000002], host_ids=[3000001, 3000002, 3000003], zone1.host_ids=[3000001], zone_names=[zone1, zone2, zone3]}}, executor=192.168.8.100 2024-05-18 20:38:26.047 INFO 11 --- [pool-manual-subtask-executor6,b64e037e7e37490f,07e60bc8b48b] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 3000003 2024-05-18 20:38:26.063 INFO 11 --- [pool-manual-subtask-executor6,b64e037e7e37490f,07e60bc8b48b] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://192.168.8.132:62888/api/v1/ob/observer/start, request body:StartObServerProcessRequest(obPath=ObPath(installPath=/home/admin/oceanbase, dataPath=/data/1, logPath=/data/log1), withArguments=true, arguments=StartObServerProcessArguments(serverIp=192.168.8.132, zoneName=zone3, configUrl=http://192.168.8.100:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster1, rsList=192.168.8.130:2882:2881;192.168.8.131:2882:2881;192.168.8.132:2882:2881, clusterName=obcluster1, obClusterId=1715516302, sqlPort=2881, svrPort=2882, optionString=datafile_disk_percentage=50,system_memory=10G), readOnly=false), params:null 2024-05-18 20:38:33.996 INFO 11 --- [pool-manual-subtask-executor6,b64e037e7e37490f,07e60bc8b48b] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 3000002 2024-05-18 20:38:34.013 INFO 11 --- [pool-manual-subtask-executor6,b64e037e7e37490f,07e60bc8b48b] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://192.168.8.131:62888/api/v1/ob/observer/start, request body:StartObServerProcessRequest(obPath=ObPath(installPath=/home/admin/oceanbase, dataPath=/data/1, logPath=/data/log1), withArguments=true, arguments=StartObServerProcessArguments(serverIp=192.168.8.131, zoneName=zone2, configUrl=http://192.168.8.100:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster1, rsList=192.168.8.130:2882:2881;192.168.8.131:2882:2881;192.168.8.132:2882:2881, clusterName=obcluster1, obClusterId=1715516302, sqlPort=2881, svrPort=2882, optionString=datafile_disk_percentage=50,system_memory=10G), readOnly=false), params:null 2024-05-18 20:38:59.859 WARN 11 --- [pool-manual-subtask-executor6,b64e037e7e37490f,07e60bc8b48b] c.o.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceOverview{id=3006037, name=Bootstrap ob, state=FAILED, operation=EXECUTE, className=com.oceanbase.ocp.service.task.business.cluster.BootStrapObTask, seriesId=25, startTime=2024-05-18T20:38:25.879+08:00, endTime=null} com.oceanbase.ocp.executor.exception.HttpRequestFailedException: [AgentClient]:http request is failed, response:AgentResponse(successful=false, timestamp=2024-05-18T20:39:08.532255678+08:00, duration=25786, status=Internal Server Error, traceId=b64e037e7e37490f, server=192.168.8.131, data=null, error=code=1500,message=Execute shell command failed: failed to execute shell command:cd /home/admin/oceanbase; ulimit -s 10240; ulimit -c unlimited; LD_LIBRARY_PATH=/home/admin/oceanbase/lib:$LD_LIBRARY_PATH LD_PRELOAD='' /home/admin/oceanbase/bin/observer -i ens33 -p 2881 -P 2882 -n obcluster1 -z zone2 -d /home/admin/oceanbase/store/obcluster1 -l info -o'obconfig_url=http://192.168.8.100:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster1,rootservice_list=192.168.8.130:2882:2881;192.168.8.131:2882:2881;192.168.8.132:2882:2881,config_additional_dir=/data/log1/obcluster1/etc2,/data/1/obcluster1/etc3,cluster_id=1715516302,datafile_disk_percentage=50,system_memory=10G', err:Command timed out., output: Session terminated, killing shell... ...killed. /home/admin/oceanbase/bin/observer -i ens33 -p 2881 -P 2882 -n obcluster1 -z zone2 -d /home/admin/oceanbase/store/obcluster1 -l info -oobconfig_url=http://192.168.8.100:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster1,rootservice_list=192.168.8.130:2882:2881;192.168.8.131:2882:2881;192.168.8.132:2882:2881,config_additional_dir=/data/log1/obcluster1/etc2,/data/1/obcluster1/etc3,cluster_id=1715516302,datafile_disk_percentage=50,system_memory=10G devname: ens33 mysql port: 2881 rpc port: 2882 appname: obcluster1 zone: zone2 data_dir: /home/admin/oceanbase/store/obcluster1 log level: info optstr: obconfig_url=http://192.168.8.100:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster1,rootservice_list=192.168.8.130:2882:2881;192.168.8.131:2882:2881;192.168.8.132:2882:2881,config_additional_dir=/data/log1/obcluster1/etc2,/data/1/obcluster1/etc3,cluster_id=1715516302,datafile_disk_percentage=50,system_memory=10G ,debugMessage=,subErrors=,) at com.oceanbase.ocp.executor.internal.template.HttpTemplate.checkSuccess(HttpTemplate.java:391) ~[command-executor-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.executor.internal.template.HttpTemplate.doPost(HttpTemplate.java:248) ~[command-executor-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.executor.internal.template.HttpTemplate.postForNone(HttpTemplate.java:116) ~[command-executor-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.executor.executor.AgentExecutor.startObServerProcess(AgentExecutor.java:298) ~[command-executor-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.obops.internal.cluster.ClusterHostOperationServiceImpl.startObServerProcess(ClusterHostOperationServiceImpl.java:256) ~[ocp-obops-service-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.service.oceanbase.ObOperationManager.startObServerProcessWithParam(ObOperationManager.java:67) ~[ocp-service-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.service.task.business.cluster.BootStrapObTask.lambda$run$9(BootStrapObTask.java:134) ~[ocp-service-4.0.3-20230331.jar!/:4.0.3-20230331] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[na:1.8.0_312] at com.oceanbase.ocp.service.task.business.cluster.BootStrapObTask.run(BootStrapObTask.java:133) ~[ocp-service-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.execute(JavaSubtaskRunner.java:60) ~[ocp-core-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(Ja vaSubtaskRunner.java:32) ~[ocp-core-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) ~[ocp-core-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:111) ~[ocp-core-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:183) ~[ocp-core-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:101) ~[ocp-core-4.0.3-20230331.jar!/:4.0.3-20230331] at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$null$1(ReadySubtaskWorker.java:127) ~[ocp-core-4.0.3-20230331.jar!/:4.0.3-20230331] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Set state for subtask: 3006037, operation:EXECUTE, state: FAILED