2024-10-31 09:36:58.048 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.t.e.runner.JavaSubtaskRunner : Run subtask, id=9896633, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/app/oceanbase/userdb/log, task_instance_id=9719362, z1.idc=pengboshi, task_operation=execute, ob_run_path=/app/oceanbase/userdb/product/oceanbase/oceanbase, cluster_type=PRIMARY, ob_install_path=/app/oceanbase/userdb/product/oceanbase/oceanbase, obproxy_proxyro_username=proxyro, ob_sql_port=2881, link_obproxy=false, primary_zone=z1;z2,z3, cluster_id=2000005, operation_scope=Cluster, ob_svr_port=2882, sub_task_instance_name=Bootstrap ob, sub_task_instance_id=9896633, cluster_version=3.1.5, cluster_name=obexam, target_server_status=RUNNING, mix_rpm_names={"z1":"oceanbase-ce-3.1.5-100000252023041721.el7.x86_64.rpm","z2":"oceanbase-ce-3.1.5-100000252023041721.el7.x86_64.rpm","z3":"oceanbase-ce-3.1.5-100000252023041721.el7.x86_64.rpm"}, startup_option_string=memory_limit=10G,system_memory=2G,cpu_count=4,cache_wash_threshold=1GB,net_thread_count=4,workers_per_cpu_quota=2,stack_size=512k, subtask_splitter=host_ids, ob_version=3.1.5, service_name=obexam:1712780980, target_zone_status=RUNNING, z3.idc=pengboshi, ob_cluster_id=1712780980, service_type=OB_CLUSTER, obproxy_proxyro_password=******, root_sys_password=******, z2.idc=pengboshi, dep_rpm_names={"z1":["oceanbase-ce-libs-3.1.5-100000252023041721.el7.x86_64.rpm","oceanbase-ce-utils-3.1.5-100000252023041721.el7.x86_64.rpm"],"z2":["oceanbase-ce-libs-3.1.5-100000252023041721.el7.x86_64.rpm","oceanbase-ce-utils-3.1.5-100000252023041721.el7.x86_64.rpm"],"z3":["oceanbase-ce-libs-3.1.5-100000252023041721.el7.x86_64.rpm","oceanbase-ce-utils-3.1.5-100000252023041721.el7.x86_64.rpm"]}, ob_run_user=ob, target_cluster_status=RUNNING, latest_execution_start_time=2024-10-31T09:36:57.933+08:00, ob_data_disk_path=/app/oceanbase/userdb/data, mix_obs_rpm={}}, listMap={root_servers=[3000001, 3000002, 3000003], z3.host_ids=[3000003], server_ids=[3000013, 300001 4, 3000015], z1.host_ids=[3000001], host_ids=[3000001, 3000002, 3000003], z2.host_ids=[3000002], zone_names=[z1, z2, z3]}}, executor=172.30.100.131 2024-10-31 09:36:58.120 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=3000001 2024-10-31 09:36:58.123 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 3000001 2024-10-31 09:36:58.137 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.128:62888/api/v1/ob/observer/start, request body:StartObServerProcessRequest(obPath=ObPath(installPath=/app/oceanbase/userdb/product/oceanbase/oceanbase, dataPath=/app/oceanbase/userdb/data, logPath=/app/oceanbase/userdb/log, diskPathStyle=null, runPath=/app/oceanbase/userdb/product/oceanbase/oceanbase), withArguments=true, arguments=StartObServerProcessArguments(serverIp=172.30.100.128, ipAddressType=DEVICE_NAME, zoneName=z1, configUrl=http://172.30.100.131:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obexam, rsList=172.30.100.128:2882:2881;172.30.100.129:2882:2881;172.30.100.130:2882:2881, clusterName=obexam, obClusterId=1712780980, sqlPort=2881, svrPort=2882, optionString=memory_limit=10G,system_memory=2G,cpu_count=4,cache_wash_threshold=1GB,net_thread_count=4,workers_per_cpu_quota=2,stack_size=512k), runUser=ob, runGroup=null, readOnly=false), params:null 2024-10-31 09:36:58.273 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=3000002 2024-10-31 09:36:58.275 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 3000002 2024-10-31 09:36:58.290 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.129:62888/api/v1/ob/observer/start, request body:StartObServerProcessRequest(obPath=ObPath(installPath=/app/oceanbase/userdb/product/oceanbase/oceanbase, dataPath=/app/oceanbase/userdb/data, logPath=/app/oceanbase/userdb/log, diskPathStyle=null, runPath=/app/oceanbase/userdb/product/oceanbase/oceanbase), withArguments=true, arguments=StartObServerProcessArguments(serverIp=172.30.100.129, ipAddressType=DEVICE_NAME, zoneName=z2, configUrl=http://172.30.100.131:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obexam, rsList=172.30.100.128:2882:2881;172.30.100.129:2882:2881;172.30.100.130:2882:2881, clusterName=obexam, obClusterId=1712780980, sqlPort=2881, svrPort=2882, optionString=memory_limit=10G,system_memory=2G,cpu_count=4,cache_wash_threshold=1GB,net_thread_count=4,workers_per_cpu_quota=2,stack_size=512k), runUser=ob, runGroup=null, readOnly=false), params:null 2024-10-31 09:36:58.423 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=3000003 2024-10-31 09:36:58.426 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 3000003 2024-10-31 09:36:58.443 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.130:62888/api/v1/ob/observer/start, request body:StartObServerProcessRequest(obPath=ObPath(installPath=/app/oceanbase/userdb/product/oceanbase/oceanbase, dataPath=/app/oceanbase/userdb/data, logPath=/app/oceanbase/userdb/log, diskPathStyle=null, runPath=/app/oceanbase/userdb/product/oceanbase/oceanbase), withArguments=true, arguments=StartObServerProcessArguments(serverIp=172.30.100.130, ipAddressType=DEVICE_NAME, zoneName=z3, configUrl=http://172.30.100.131:8080/services?Action=ObRootServiceInfo&User_ID=alibaba&UID=ocpmaster&ObRegion=obexam, rsList=172.30.100.128:2882:2881;172.30.100.129:2882:2881;172.30.100.130:2882:2881, clusterName=obexam, obClusterId=1712780980, sqlPort=2881, svrPort=2882, optionString=memory_limit=10G,system_memory=2G,cpu_count=4,cache_wash_threshold=1GB,net_thread_count=4,workers_per_cpu_quota=2,stack_size=512k), runUser=ob, runGroup=null, readOnly=false), params:null 2024-10-31 09:36:58.585 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.s.t.b.c.helper.ObServerTaskHelper : begin to check server accessible, hostIds=[3000001, 3000002, 3000003] 2024-10-31 09:36:58.602 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=3000001 2024-10-31 09:36:58.605 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 3000001 2024-10-31 09:36:58.622 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.128:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null 2024-10-31 09:36:58.625 WARN 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.s.t.b.c.helper.ObServerTaskHelper : Failed to check observer accessible, reason:[AgentClient]:http request is failed, response:Unexpected error: dial tcp 127.0.0.1:2881: connect: connection refused, cause:null 2024-10-31 09:36:58.627 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.ocp.common.lang.pattern.Retry : wait for 5 seconds 2024-10-31 09:37:03.648 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=3000001 2024-10-31 09:37:03.697 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 3000001 2024-10-31 09:37:03.774 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.128:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null 2024-10-31 09:37:04.185 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=3000002 2024-10-31 09:37:04.188 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 3000002 2024-10-31 09:37:04.204 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.129:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null 2024-10-31 09:37:04.585 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=3000003 2024-10-31 09:37:04.588 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 3000003 2024-10-31 09:37:04.604 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.130:62888/api/v1/ob/observer/access, request body:AccessObServerProcessRequest(port=2881, username=root), params:null 2024-10-31 09:37:04.960 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.s.t.b.c.helper.ObServerTaskHelper : all ob server accessible 2024-10-31 09:37:04.961 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.s.t.b.cluster.BootStrapObTask : bootstrap request: BootstrapRequest(clusterType=PRIMARY, rsInfoList=[RsInfo(region=chaoyang, zone=z1, server=ServerInfo(ip=172.30.100.128, port=2882)), RsInfo(region=chaoyang, zone=z2, server=ServerInfo(ip=172.30.100.129, port=2882)), RsInfo(region=chaoyang, zone=z3, server=ServerInfo(ip=172.30.100.130, port=2882))], primaryInfo=null, sqlPort=2881, timeoutSeconds=1200) 2024-10-31 09:37:04.967 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=3000001 2024-10-31 09:37:04.970 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 3000001 2024-10-31 09:37:04.986 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.128:62888/api/v1/ob/bootstrap, request body:BootstrapRequest(clusterType=PRIMARY, rsInfoList=[RsInfo(region=chaoyang, zone=z1, server=ServerInfo(ip=172.30.100.128, port=2882)), RsInfo(region=chaoyang, zone=z2, server=ServerInfo(ip=172.30.100.129, port=2882)), RsInfo(region=chaoyang, zone=z3, server=ServerInfo(ip=172.30.100.130, port=2882))], primaryInfo=null, sqlPort=2881, timeoutSeconds=1200), params:null 2024-10-31 09:37:04.990 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.128:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=5e8b0187-b7b8-49ce-9772-0076e55ff75d), params:null 2024-10-31 09:37:04.992 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2024-10-31 09:37:04.994 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2024-10-31 09:37:19.995 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.128:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=5e8b0187-b7b8-49ce-9772-0076e55ff75d), params:null 2024-10-31 09:37:20.002 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2024-10-31 09:37:20.004 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2024-10-31 09:37:35.006 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.30.100.128:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=5e8b0187-b7b8-49ce-9772-0076e55ff75d), params:null 2024-10-31 09:37:35.010 INFO 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,OcpAgentError: code = 1002, message = Unexpected error: Error 1235 (0A000): unit min memory less than __min_full_resource_pool_memory not supported, 2024-10-31 09:37:35.012 ERROR 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.s.task.util.AgentAsyncTaskHelper : failed to execute async task, task token: 5e8b0187-b7b8-49ce-9772-0076e55ff75d, err: OcpAgentError: code = 1002, message = Unexpected error: Error 1235 (0A000): unit min memory less than __min_full_resource_pool_memory not supported 2024-10-31 09:37:35.014 ERROR 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.ocp.core.util.ExceptionUtils : Checked Exception: com.oceanbase.ocp.core.exception.UnexpectedException occurred with code error.agent.task.failed, and args [5e8b0187-b7b8-49ce-9772-0076e55ff75d, OcpAgentError: code = 1002, message = Unexpected error: Error 1235 (0A000): unit min memory less than __min_full_resource_pool_memory not supported] 2024-10-31 09:37:35.017 ERROR 5290 --- [pool-manual-subtask-executor13,728f3356067e4376,e4c16336b7b6] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : OCP-Agent task 5e8b0187-b7b8-49ce-9772-0076e55ff75d failed, error message: OcpAgentError: code = 1002, message = Unexpected error: Error 1235 (0A000): unit min memory less than __min_full_resource_pool_memory not supported com.oceanbase.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=AGENT_TASK_FAILED, args=5e8b0187-b7b8-49ce-9772-0076e55ff75d,OcpAgentError: code = 1002, message = Unexpected error: Error 1235 (0A000): unit min memory less than __min_full_resource_pool_memory not supported at com.oceanbase.ocp.core.i18n.ErrorCodes$Kind$10.exception(ErrorCodes.java:1570) at com.oceanbase.ocp.core.i18n.ErrorCodes.exception(ErrorCodes.java:1425) at com.oceanbase.ocp.core.util.ExceptionUtils.newException(ExceptionUtils.java:169) at com.oceanbase.ocp.core.util.ExceptionUtils.throwException(ExceptionUtils.java:162) at com.oceanbase.ocp.service.task.util.AgentAsyncTaskHelper.checkSuccess(AgentAsyncTaskHelper.java:229) at com.oceanbase.ocp.service.task.util.AgentAsyncTaskHelper.waitForExecuteFinish(AgentAsyncTaskHelper.java:175) at com.oceanbase.ocp.service.task.util.AgentAsyncTaskHelper.bootstrapOb(AgentAsyncTaskHelper.java:119) at com.oceanbase.ocp.service.task.business.cluster.BootStrapObTask.run(BootStrapObTask.java:171) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.execute(JavaSubtaskRunner.java:64) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:32) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutp ut(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 9896633, operation:EXECUTE, state: FAILED