2022-06-22 20:09:01.209 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.o.c.m.j.model.SubtaskInstanceEntity : Retry subtask, id=540, context=Context(parallelIdx=0, stringMap={cluster_name=obcluster, target_server_status=RUNNING, startup_option_string=, service_name=obcluster:1, target_zone_status=RUNNING, ob_log_disk_path=/TRS/ocearbase/observer/store, task_instance_id=453, task_operation=retry, service_version=3.1.3, ob_cluster_id=1, ob_install_path=/TRS/ocearbase/observer, host.4.zone=trszone1, cluster_id=1, service_type=OB_CLUSTER, rpm_name=oceanbase-ce-3.1.3-10100032022041510.el7.x86_64.rpm, target_cluster_status=RUNNING, latest_execution_start_time=2022-06-22T20:09:01.185+08:00, host.5.zone=trszone1, ob_data_disk_path=/TRS/ocearbase/observer/store, sub_task_instance_id=540}, listMap={root_server_ips=[10.12.71.16], server_ids=[4, 5], dep_rpm_names=[oceanbase-ce-libs-3.1.3-10100032022041510.el7.x86_64.rpm], exists_server_addrs=[10.12.71.16:2882], host_ids=[5, 4], zone_names=[trszone1]}), executor=10.12.71.16 2022-06-22 20:09:01.212 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] .t.b.s.StartObServerProcessWithParamTask : begin to rollback 2022-06-22 20:09:01.234 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] svrIp=10.12.71.18, port=62888, user=root, cmd=pgrep -c observer | xargs --no-run-if-empty 2022-06-22 20:09:01.294 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] result=0 2022-06-22 20:09:01.301 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.o.service.compute.host.HostManager : result=0 2022-06-22 20:09:01.304 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.o.s.t.b.server.ObServerTaskHandler : no need to stop, observer process is not exist 2022-06-22 20:09:01.321 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] svrIp=10.12.71.18, port=62888, user=root, cmd=pgrep -c observer | xargs --no-run-if-empty 2022-06-22 20:09:01.379 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] result=0 2022-06-22 20:09:01.383 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.o.service.compute.host.HostManager : result=0 2022-06-22 20:09:01.398 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] svrIp=10.12.71.18, port=62888, user=root, cmd=rm -rf /TRS/ocearbase/observer/store/obcluster/*/* 2022-06-22 20:09:01.447 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] result= 2022-06-22 20:09:01.454 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] .t.b.s.StartObServerProcessWithParamTask : rollback finished 2022-06-22 20:09:01.468 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] .t.b.s.StartObServerProcessWithParamTask : begin to start observer process with param 2022-06-22 20:09:01.492 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] svrIp=10.12.71.18, port=62888, user=root, cmd=ifconfig 2> /dev/null | grep '10\.12\.71\.18' -B 1 | grep -E 'flags|HWaddr' | awk -F '[: ]+' '{print $1}' 2022-06-22 20:09:01.536 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] result=eth0 2022-06-22 20:09:01.550 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] svrIp=10.12.71.18, port=62888, user=admin, cmd=cd /TRS/ocearbase/observer; ulimit -s 10240; ulimit -c unlimited; LD_LIBRARY_PATH=/TRS/ocearbase/observer/lib:$LD_LIBRARY_PATH LD_PRELOAD='' /TRS/ocearbase/observer/bin/observer -i eth0 -p 2881 -P 2882 -n obcluster -z trszone1 -d /TRS/ocearbase/observer/store/obcluster -l info -o'obconfig_url=http://10.12.71.16:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster,rootservice_list=10.12.71.16:2882:2881,config_additional_dir=/TRS/ocearbase/observer/store/obcluster/etc2,/TRS/ocearbase/observer/store/obcluster/etc3,cluster_id=1,' 2022-06-22 20:09:01.674 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] result=/TRS/ocearbase/observer/bin/observer -i eth0 -p 2881 -P 2882 -n obcluster -z trszone1 -d /TRS/ocearbase/observer/store/obcluster -l info -oobconfig_url=http://10.12.71.16:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster,rootservice_list=10.12.71.16:2882:2881,config_additional_dir=/TRS/ocearbase/observer/store/obcluster/etc2,/TRS/ocearbase/observer/store/obcluster/etc3,cluster_id=1, devname: eth0 mysql port: 2881 rpc port: 2882 appname: obcluster zone: trszone1 data_dir: /TRS/ocearbase/observer/store/obcluster log level: info optstr: obconfig_url=http://10.12.71.16:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obcluster,rootservice_list=10.12.71.16:2882:2881,config_additional_dir=/TRS/ocearbase/observer/store/obcluster/etc2,/TRS/ocearbase/observer/store/obcluster/etc3,cluster_id=1, 2022-06-22 20:09:01.677 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] com.alipay.ocp.core.util.TaskUtils : [TaskUtils] wait 60 seconds 2022-06-22 20:10:01.694 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] svrIp=10.12.71.18, port=62888, user=root, cmd=pgrep -c observer | xargs --no-run-if-empty 2022-06-22 20:10:01.760 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.task.util.OcpAgentUtils : [OcpAgentUtils.runCmd] result=0 2022-06-22 20:10:01.765 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.o.service.compute.host.HostManager : result=0 2022-06-22 20:10:01.768 ERROR 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] com.alipay.ocp.core.util.ExceptionUtils : Checked Exception: com.alipay.ocp.core.exception.UnexpectedException occurred with code error.common.unexpected, and args [no observer process] 2022-06-22 20:10:01.771 INFO 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.o.c.m.j.model.SubtaskInstanceEntity : Set state for subtask: 540, current state: RUNNING, new state: FAILED 2022-06-22 20:10:01.778 WARN 35 --- [pool-subtask-executor-thread-27,3b1bbb9794db4ba1,6c04c51f0e13] c.a.ocp.core.job.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=540, name=Start observer process with param, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.server.StartObServerProcessWithParamTask, seriesId=0, startTime=2022-06-22T20:09:01.185+08:00, endTime=2022-06-22T20:10:01.777+08:00}, failedMessage=An unknown error has occurred. Cause: no observer process. Error message: {1}. Contact the administrator. com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=COMMON_UNEXPECTED, args=no observer process at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[na:1.8.0_312] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[na:1.8.0_312] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[na:1.8.0_312] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[na:1.8.0_312] at com.alipay.ocp.core.util.ExceptionUtils.newException(ExceptionUtils.java:96) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.core.util.ExceptionUtils.throwException(ExceptionUtils.java:90) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.core.util.ExceptionUtils.unExpected(ExceptionUtils.java:71) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.service.task.business.server.StartObServerProcessWithParamTask.run(StartObServerProcessWithParamTask.java:75) ~[ocp-service-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.core.task.task.ITask.retry(ITask.java:45) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.core.metadb.job.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:225) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.core.job.runner.JavaTaskRunner.doExecute(JavaTaskRunn er.java:30) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.core.job.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.core.job.runner.RunnerFactory.doRun(RunnerFactory.java:103) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.core.job.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:147) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.core.job.runner.RunnerFactory.run(RunnerFactory.java:92) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at com.alipay.ocp.core.job.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$2(ReadySubtaskWorker.java:123) ~[ocp-core-3.1.1-20210916.jar!/:3.1.1-20210916] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312]