############{EXECUTE}{2025-03-31T22:33:41.729+08:00}############ 2025-03-31 22:33:41.734 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.ocp.service.iam.user.UserService : user 1001002 login with organization 10000000 2025-03-31 22:33:41.737 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.c.t.e.runner.JavaSubtaskRunner : Run subtask, id=9171685, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/data/log1, primary_root_sys_password=xxx task_instance_id=7941627, task_operation=execute, ob_run_path=/home/admin/oceanbase, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, obrs.1711351485.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"ten001\":\"True\",\"ten002\":\"True\",\"ten003\":\"True\",\"ten004\":\"True\",\"ten010\":\"True\",\"ten011\":\"True\",\"odc_meta\":\"True\",\"oms_meta\":\"True\",\"sys\":\"True\",\"ten005\":\"True\",\"ten006\":\"True\",\"ten007\":\"True\"}","enable_rereplication":"True"}, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade health checker script, watched_scenario=UPGRADE, sub_task_instance_id=9171685, primary_cluster_id=2, cluster_version=4.2.2.1, cluster_name=obrs, mix_rpm_names={"zone3":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1711351485, target_version=4.2.5.2-102010012025022610, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2025-03-31T22:33:41.724+08:00, ob_data_disk_path=/data/1}, listMap={standby_cluster_ids=[], 1711351485.obrs.zone3.server_ids=[4], 1711351485.o brs.zone1.server_ids=[2], cluster_ids=[2], primary_first_cluster_ids=[2], 1711351485.obrs.zone3.host_ids=[4], standby_first_cluster_ids=[2], 1711351485.obrs.zone2.host_ids=[3], staged_versions=[4.2.5.2-102010012025022610], 1711351485.obrs.zone2.server_ids=[3], direct_upgrade_versions=[], 1711351485.obrs.zone_names=[zone1, zone2, zone3], 1711351485.obrs.zone1.host_ids=[2], zone_names=[zone1, zone2, zone3], server_ids=[2, 3, 4], host_ids=[2, 3, 4]}}, executor=10.10.10.39 2025-03-31 22:33:41.767 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obrs, tenantName=sys, dbUser=root 2025-03-31 22:33:41.770 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:33:41.773 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2025-03-31 22:33:41.785 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:33:41.788 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2025-03-31 22:33:41.797 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:33:41.800 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2025-03-31 22:33:41.808 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py on cluster, clusterId=2, obClusterId=1711351485, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm 2025-03-31 22:33:41.814 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:33:41.818 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:33:41.833 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py), params:null 2025-03-31 22:33:41.838 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:e3a723b0-d038-447f-adbb-aebae94017e1 2025-03-31 22:33:41.843 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:33:41.846 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:33:41.861 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.5.2-102010012025022610_upgrade_health_checker_20250331223341.log, zone=null, cmdTimeout=0)), params:null 2025-03-31 22:33:41.892 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=e3a723b0-d038-447f-adbb-aebae94017e1), params:null 2025-03-31 22:33:41.895 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2025-03-31 22:33:41.898 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2025-03-31 22:33:56.900 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=e3a723b0-d038-447f-adbb-aebae94017e1), params:null 2025-03-31 22:33:56.905 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:true,null,, 2025-03-31 22:33:56.908 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,35ecd2eb5c33e198] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py finished Set state for subtask: 9171685, operation:EXECUTE, state: SUCCESSFUL Set state for subtask: 9171685, operation:EXECUTE, state: FAILED ############{RETRY}{2025-03-31T22:35:08.956+08:00}############ 2025-03-31 22:35:08.963 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.ocp.service.iam.user.UserService : user 1001002 login with organization 10000000 2025-03-31 22:35:08.966 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=9171685, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/data/log1, primary_root_sys_password=xxx task_instance_id=7941627, task_operation=retry, ob_run_path=/home/admin/oceanbase, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, obrs.1711351485.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"ten001\":\"True\",\"ten002\":\"True\",\"ten003\":\"True\",\"ten004\":\"True\",\"ten010\":\"True\",\"ten011\":\"True\",\"odc_meta\":\"True\",\"oms_meta\":\"True\",\"sys\":\"True\",\"ten005\":\"True\",\"ten006\":\"True\",\"ten007\":\"True\"}","enable_rereplication":"True"}, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade health checker script, watched_scenario=UPGRADE, sub_task_instance_id=9171685, primary_cluster_id=2, cluster_version=4.2.2.1, cluster_name=obrs, mix_rpm_names={"zone3":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1711351485, target_version=4.2.5.2-102010012025022610, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2025-03-31T22:35:08.948+08:00, ob_data_disk_path=/data/1}, listMap={standby_cluster_ids=[], 1711351485.obrs.zone3.server_ids=[4], 1711351485.o brs.zone1.server_ids=[2], cluster_ids=[2], primary_first_cluster_ids=[2], 1711351485.obrs.zone3.host_ids=[4], standby_first_cluster_ids=[2], 1711351485.obrs.zone2.host_ids=[3], staged_versions=[4.2.5.2-102010012025022610], 1711351485.obrs.zone2.server_ids=[3], direct_upgrade_versions=[], 1711351485.obrs.zone_names=[zone1, zone2, zone3], 1711351485.obrs.zone1.host_ids=[2], zone_names=[zone1, zone2, zone3], server_ids=[2, 3, 4], host_ids=[2, 3, 4]}}, executor=10.10.10.39 2025-03-31 22:35:08.995 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obrs, tenantName=sys, dbUser=root 2025-03-31 22:35:08.998 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:35:09.001 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2025-03-31 22:35:09.014 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:35:09.016 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2025-03-31 22:35:09.032 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:35:09.035 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2025-03-31 22:35:09.038 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py on cluster, clusterId=2, obClusterId=1711351485, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm 2025-03-31 22:35:09.044 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:35:09.047 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:35:09.064 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py), params:null 2025-03-31 22:35:09.070 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:92f504a6-653f-475e-aabd-1403a195280e 2025-03-31 22:35:09.076 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:35:09.080 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:35:09.097 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.5.2-102010012025022610_upgrade_health_checker_20250331223509.log, zone=null, cmdTimeout=0)), params:null 2025-03-31 22:35:09.134 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=92f504a6-653f-475e-aabd-1403a195280e), params:null 2025-03-31 22:35:09.137 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2025-03-31 22:35:09.139 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2025-03-31 22:35:24.144 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=92f504a6-653f-475e-aabd-1403a195280e), params:null 2025-03-31 22:35:24.150 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:true,null,, 2025-03-31 22:35:24.153 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,6e5b9bc418a79b14] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py finished Set state for subtask: 9171685, operation:RETRY, state: SUCCESSFUL Set state for subtask: 9171685, operation:RETRY, state: FAILED ############{RETRY}{2025-03-31T22:38:45.464+08:00}############ 2025-03-31 22:38:45.472 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.ocp.service.iam.user.UserService : user 1001002 login with organization 10000000 2025-03-31 22:38:45.482 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=9171685, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/data/log1, primary_root_sys_password=xxx task_instance_id=7941627, task_operation=retry, ob_run_path=/home/admin/oceanbase, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, obrs.1711351485.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"ten001\":\"True\",\"ten002\":\"True\",\"ten003\":\"True\",\"ten004\":\"True\",\"ten010\":\"True\",\"ten011\":\"True\",\"odc_meta\":\"True\",\"oms_meta\":\"True\",\"sys\":\"True\",\"ten005\":\"True\",\"ten006\":\"True\",\"ten007\":\"True\"}","enable_rereplication":"True"}, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade health checker script, watched_scenario=UPGRADE, sub_task_instance_id=9171685, primary_cluster_id=2, cluster_version=4.2.2.1, cluster_name=obrs, mix_rpm_names={"zone3":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1711351485, target_version=4.2.5.2-102010012025022610, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2025-03-31T22:38:45.455+08:00, ob_data_disk_path=/data/1}, listMap={standby_cluster_ids=[], 1711351485.obrs.zone3.server_ids=[4], 1711351485.o brs.zone1.server_ids=[2], cluster_ids=[2], primary_first_cluster_ids=[2], 1711351485.obrs.zone3.host_ids=[4], standby_first_cluster_ids=[2], 1711351485.obrs.zone2.host_ids=[3], staged_versions=[4.2.5.2-102010012025022610], 1711351485.obrs.zone2.server_ids=[3], direct_upgrade_versions=[], 1711351485.obrs.zone_names=[zone1, zone2, zone3], 1711351485.obrs.zone1.host_ids=[2], zone_names=[zone1, zone2, zone3], server_ids=[2, 3, 4], host_ids=[2, 3, 4]}}, executor=10.10.10.39 2025-03-31 22:38:45.511 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obrs, tenantName=sys, dbUser=root 2025-03-31 22:38:45.514 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:38:45.516 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2025-03-31 22:38:45.530 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:38:45.532 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2025-03-31 22:38:45.549 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:38:45.552 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2025-03-31 22:38:45.555 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py on cluster, clusterId=2, obClusterId=1711351485, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm 2025-03-31 22:38:45.560 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:38:45.564 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:38:45.578 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py), params:null 2025-03-31 22:38:45.583 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:db3086e4-ea43-40f0-a17f-91a148521a6b 2025-03-31 22:38:45.589 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:38:45.592 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:38:45.608 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.5.2-102010012025022610_upgrade_health_checker_20250331223845.log, zone=null, cmdTimeout=0)), params:null 2025-03-31 22:38:45.641 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=db3086e4-ea43-40f0-a17f-91a148521a6b), params:null 2025-03-31 22:38:45.644 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2025-03-31 22:38:45.647 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2025-03-31 22:39:00.649 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=db3086e4-ea43-40f0-a17f-91a148521a6b), params:null 2025-03-31 22:39:00.654 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:true,null,, 2025-03-31 22:39:00.657 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,c6721b8cd22830b7] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py finished Set state for subtask: 9171685, operation:RETRY, state: SUCCESSFUL Set state for subtask: 9171685, operation:RETRY, state: FAILED ############{RETRY}{2025-03-31T22:41:23.871+08:00}############ 2025-03-31 22:41:23.877 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.ocp.service.iam.user.UserService : user 1001002 login with organization 10000000 2025-03-31 22:41:23.881 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=9171685, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/data/log1, primary_root_sys_password=xxx task_instance_id=7941627, task_operation=retry, ob_run_path=/home/admin/oceanbase, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, obrs.1711351485.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"ten001\":\"True\",\"ten002\":\"True\",\"ten003\":\"True\",\"ten004\":\"True\",\"ten010\":\"True\",\"ten011\":\"True\",\"odc_meta\":\"True\",\"oms_meta\":\"True\",\"sys\":\"True\",\"ten005\":\"True\",\"ten006\":\"True\",\"ten007\":\"True\"}","enable_rereplication":"True"}, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade health checker script, watched_scenario=UPGRADE, sub_task_instance_id=9171685, primary_cluster_id=2, cluster_version=4.2.2.1, cluster_name=obrs, mix_rpm_names={"zone3":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1711351485, target_version=4.2.5.2-102010012025022610, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2025-03-31T22:41:23.862+08:00, ob_data_disk_path=/data/1}, listMap={standby_cluster_ids=[], 1711351485.obrs.zone3.server_ids=[4], 1711351485.o brs.zone1.server_ids=[2], cluster_ids=[2], primary_first_cluster_ids=[2], 1711351485.obrs.zone3.host_ids=[4], standby_first_cluster_ids=[2], 1711351485.obrs.zone2.host_ids=[3], staged_versions=[4.2.5.2-102010012025022610], 1711351485.obrs.zone2.server_ids=[3], direct_upgrade_versions=[], 1711351485.obrs.zone_names=[zone1, zone2, zone3], 1711351485.obrs.zone1.host_ids=[2], zone_names=[zone1, zone2, zone3], server_ids=[2, 3, 4], host_ids=[2, 3, 4]}}, executor=10.10.10.39 2025-03-31 22:41:23.911 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obrs, tenantName=sys, dbUser=root 2025-03-31 22:41:23.913 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:41:23.916 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2025-03-31 22:41:23.930 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:41:23.933 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2025-03-31 22:41:23.947 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:41:23.951 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2025-03-31 22:41:23.956 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py on cluster, clusterId=2, obClusterId=1711351485, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm 2025-03-31 22:41:23.962 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:41:23.967 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:41:23.984 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py), params:null 2025-03-31 22:41:23.989 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:029d444a-2369-44cf-8df5-b2845f66bd00 2025-03-31 22:41:23.995 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:41:23.998 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:41:24.015 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.5.2-102010012025022610_upgrade_health_checker_20250331224123.log, zone=null, cmdTimeout=0)), params:null 2025-03-31 22:41:24.043 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=029d444a-2369-44cf-8df5-b2845f66bd00), params:null 2025-03-31 22:41:24.046 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2025-03-31 22:41:24.049 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2025-03-31 22:41:39.051 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=029d444a-2369-44cf-8df5-b2845f66bd00), params:null 2025-03-31 22:41:39.056 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:true,null,, 2025-03-31 22:41:39.058 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,2e97426b99b66b4c] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py finished Set state for subtask: 9171685, operation:RETRY, state: SUCCESSFUL Set state for subtask: 9171685, operation:RETRY, state: FAILED ############{RETRY}{2025-03-31T22:44:54.359+08:00}############ 2025-03-31 22:44:54.363 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.ocp.service.iam.user.UserService : user 1001002 login with organization 10000000 2025-03-31 22:44:54.375 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=9171685, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/data/log1, primary_root_sys_password=xxx task_instance_id=7941627, task_operation=retry, ob_run_path=/home/admin/oceanbase, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, obrs.1711351485.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"ten001\":\"True\",\"ten002\":\"True\",\"ten003\":\"True\",\"ten004\":\"True\",\"ten010\":\"True\",\"ten011\":\"True\",\"odc_meta\":\"True\",\"oms_meta\":\"True\",\"sys\":\"True\",\"ten005\":\"True\",\"ten006\":\"True\",\"ten007\":\"True\"}","enable_rereplication":"True"}, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade health checker script, watched_scenario=UPGRADE, sub_task_instance_id=9171685, primary_cluster_id=2, cluster_version=4.2.2.1, cluster_name=obrs, mix_rpm_names={"zone3":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1711351485, target_version=4.2.5.2-102010012025022610, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2025-03-31T22:44:54.351+08:00, ob_data_disk_path=/data/1}, listMap={standby_cluster_ids=[], 1711351485.obrs.zone3.server_ids=[4], 1711351485.o brs.zone1.server_ids=[2], cluster_ids=[2], primary_first_cluster_ids=[2], 1711351485.obrs.zone3.host_ids=[4], standby_first_cluster_ids=[2], 1711351485.obrs.zone2.host_ids=[3], staged_versions=[4.2.5.2-102010012025022610], 1711351485.obrs.zone2.server_ids=[3], direct_upgrade_versions=[], 1711351485.obrs.zone_names=[zone1, zone2, zone3], 1711351485.obrs.zone1.host_ids=[2], zone_names=[zone1, zone2, zone3], server_ids=[2, 3, 4], host_ids=[2, 3, 4]}}, executor=10.10.10.39 2025-03-31 22:44:54.402 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obrs, tenantName=sys, dbUser=root 2025-03-31 22:44:54.404 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:44:54.406 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2025-03-31 22:44:54.418 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:44:54.420 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2025-03-31 22:44:54.430 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:44:54.432 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2025-03-31 22:44:54.436 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py on cluster, clusterId=2, obClusterId=1711351485, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm 2025-03-31 22:44:54.440 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:44:54.443 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:44:54.459 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py), params:null 2025-03-31 22:44:54.464 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:6249c071-822e-4cfc-b6cf-1c9447a0e724 2025-03-31 22:44:54.468 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:44:54.472 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:44:54.486 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.5.2-102010012025022610_upgrade_health_checker_20250331224454.log, zone=null, cmdTimeout=0)), params:null 2025-03-31 22:44:54.525 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=6249c071-822e-4cfc-b6cf-1c9447a0e724), params:null 2025-03-31 22:44:54.529 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2025-03-31 22:44:54.531 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2025-03-31 22:45:09.533 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=6249c071-822e-4cfc-b6cf-1c9447a0e724), params:null 2025-03-31 22:45:09.571 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:true,null,, 2025-03-31 22:45:09.574 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,44713897433b6248] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py finished Set state for subtask: 9171685, operation:RETRY, state: SUCCESSFUL Set state for subtask: 9171685, operation:RETRY, state: FAILED ############{RETRY}{2025-03-31T22:52:24.533+08:00}############ 2025-03-31 22:52:24.539 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.ocp.service.iam.user.UserService : user 1001002 login with organization 10000000 2025-03-31 22:52:24.542 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=9171685, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/data/log1, primary_root_sys_password=xxx task_instance_id=7941627, task_operation=retry, ob_run_path=/home/admin/oceanbase, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, obrs.1711351485.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"ten001\":\"True\",\"ten002\":\"True\",\"ten003\":\"True\",\"ten004\":\"True\",\"ten010\":\"True\",\"ten011\":\"True\",\"odc_meta\":\"True\",\"oms_meta\":\"True\",\"sys\":\"True\",\"ten005\":\"True\",\"ten006\":\"True\",\"ten007\":\"True\"}","enable_rereplication":"True"}, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade health checker script, watched_scenario=UPGRADE, sub_task_instance_id=9171685, primary_cluster_id=2, cluster_version=4.2.2.1, cluster_name=obrs, mix_rpm_names={"zone3":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1711351485, target_version=4.2.5.2-102010012025022610, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2025-03-31T22:52:24.527+08:00, ob_data_disk_path=/data/1}, listMap={standby_cluster_ids=[], 1711351485.obrs.zone3.server_ids=[4], 1711351485.o brs.zone1.server_ids=[2], cluster_ids=[2], primary_first_cluster_ids=[2], 1711351485.obrs.zone3.host_ids=[4], standby_first_cluster_ids=[2], 1711351485.obrs.zone2.host_ids=[3], staged_versions=[4.2.5.2-102010012025022610], 1711351485.obrs.zone2.server_ids=[3], direct_upgrade_versions=[], 1711351485.obrs.zone_names=[zone1, zone2, zone3], 1711351485.obrs.zone1.host_ids=[2], zone_names=[zone1, zone2, zone3], server_ids=[2, 3, 4], host_ids=[2, 3, 4]}}, executor=10.10.10.39 2025-03-31 22:52:24.561 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 22:52:24.572 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obrs, tenantName=sys, dbUser=root 2025-03-31 22:52:24.574 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 22:52:24.576 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 22:52:24.578 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:52:24.581 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2025-03-31 22:52:24.596 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 22:52:24.598 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:52:24.602 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2025-03-31 22:52:24.615 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 22:52:24.618 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2025-03-31 22:52:24.622 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py on cluster, clusterId=2, obClusterId=1711351485, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm 2025-03-31 22:52:24.627 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:52:24.630 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:52:24.646 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py), params:null 2025-03-31 22:52:24.652 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:2128ffae-81d8-4e13-a74c-296d492174d1 2025-03-31 22:52:24.657 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 22:52:24.660 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 22:52:24.675 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.5.2-102010012025022610_upgrade_health_checker_20250331225224.log, zone=null, cmdTimeout=0)), params:null 2025-03-31 22:52:24.711 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=2128ffae-81d8-4e13-a74c-296d492174d1), params:null 2025-03-31 22:52:24.714 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2025-03-31 22:52:24.716 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2025-03-31 22:52:39.718 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=2128ffae-81d8-4e13-a74c-296d492174d1), params:null 2025-03-31 22:52:39.721 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:true,null,, 2025-03-31 22:52:39.724 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,f8bcb538496802f7] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py finished Set state for subtask: 9171685, operation:RETRY, state: SUCCESSFUL Set state for subtask: 9171685, operation:RETRY, state: FAILED ############{RETRY}{2025-03-31T23:02:56.169+08:00}############ 2025-03-31 23:02:56.174 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.ocp.service.iam.user.UserService : user 1001002 login with organization 10000000 2025-03-31 23:02:56.178 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=9171685, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/data/log1, primary_root_sys_password=xxx task_instance_id=7941627, task_operation=retry, ob_run_path=/home/admin/oceanbase, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, obrs.1711351485.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"ten001\":\"True\",\"ten002\":\"True\",\"ten003\":\"True\",\"ten004\":\"True\",\"ten010\":\"True\",\"ten011\":\"True\",\"odc_meta\":\"True\",\"oms_meta\":\"True\",\"sys\":\"True\",\"ten005\":\"True\",\"ten006\":\"True\",\"ten007\":\"True\"}","enable_rereplication":"True"}, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade health checker script, watched_scenario=UPGRADE, sub_task_instance_id=9171685, primary_cluster_id=2, cluster_version=4.2.2.1, cluster_name=obrs, mix_rpm_names={"zone3":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1711351485, target_version=4.2.5.2-102010012025022610, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2025-03-31T23:02:56.163+08:00, ob_data_disk_path=/data/1}, listMap={standby_cluster_ids=[], 1711351485.obrs.zone3.server_ids=[4], 1711351485.o brs.zone1.server_ids=[2], cluster_ids=[2], primary_first_cluster_ids=[2], 1711351485.obrs.zone3.host_ids=[4], standby_first_cluster_ids=[2], 1711351485.obrs.zone2.host_ids=[3], staged_versions=[4.2.5.2-102010012025022610], 1711351485.obrs.zone2.server_ids=[3], direct_upgrade_versions=[], 1711351485.obrs.zone_names=[zone1, zone2, zone3], 1711351485.obrs.zone1.host_ids=[2], zone_names=[zone1, zone2, zone3], server_ids=[2, 3, 4], host_ids=[2, 3, 4]}}, executor=10.10.10.39 2025-03-31 23:02:56.197 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:02:56.214 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obrs, tenantName=sys, dbUser=root 2025-03-31 23:02:56.217 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:02:56.219 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:02:56.221 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 23:02:56.223 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2025-03-31 23:02:56.242 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:02:56.244 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 23:02:56.247 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2025-03-31 23:02:56.264 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 23:02:56.267 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2025-03-31 23:02:56.276 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py on cluster, clusterId=2, obClusterId=1711351485, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm 2025-03-31 23:02:56.282 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 23:02:56.287 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 23:02:56.303 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py), params:null 2025-03-31 23:02:56.309 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:680ae1e6-fbd2-4b75-aa17-89f7fd6d863d 2025-03-31 23:02:56.314 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 23:02:56.318 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 23:02:56.334 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.5.2-102010012025022610_upgrade_health_checker_20250331230256.log, zone=null, cmdTimeout=0)), params:null 2025-03-31 23:02:56.438 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=680ae1e6-fbd2-4b75-aa17-89f7fd6d863d), params:null 2025-03-31 23:02:56.442 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:true,null,, 2025-03-31 23:02:56.444 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,55f96cd998ad1f4e] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py finished Set state for subtask: 9171685, operation:RETRY, state: SUCCESSFUL Set state for subtask: 9171685, operation:RETRY, state: FAILED ############{RETRY}{2025-03-31T23:16:29.184+08:00}############ 2025-03-31 23:16:29.190 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.ocp.service.iam.user.UserService : user 1001002 login with organization 10000000 2025-03-31 23:16:29.193 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=9171685, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/data/log1, primary_root_sys_password=xxx task_instance_id=7941627, task_operation=retry, ob_run_path=/home/admin/oceanbase, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, obrs.1711351485.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"ten001\":\"True\",\"ten002\":\"True\",\"ten003\":\"True\",\"ten004\":\"True\",\"ten010\":\"True\",\"ten011\":\"True\",\"odc_meta\":\"True\",\"oms_meta\":\"True\",\"sys\":\"True\",\"ten005\":\"True\",\"ten006\":\"True\",\"ten007\":\"True\"}","enable_rereplication":"True"}, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade health checker script, watched_scenario=UPGRADE, sub_task_instance_id=9171685, primary_cluster_id=2, cluster_version=4.2.2.1, cluster_name=obrs, mix_rpm_names={"zone3":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1711351485, target_version=4.2.5.2-102010012025022610, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2025-03-31T23:16:29.179+08:00, ob_data_disk_path=/data/1}, listMap={standby_cluster_ids=[], 1711351485.obrs.zone3.server_ids=[4], 1711351485.o brs.zone1.server_ids=[2], cluster_ids=[2], primary_first_cluster_ids=[2], 1711351485.obrs.zone3.host_ids=[4], standby_first_cluster_ids=[2], 1711351485.obrs.zone2.host_ids=[3], staged_versions=[4.2.5.2-102010012025022610], 1711351485.obrs.zone2.server_ids=[3], direct_upgrade_versions=[], 1711351485.obrs.zone_names=[zone1, zone2, zone3], 1711351485.obrs.zone1.host_ids=[2], zone_names=[zone1, zone2, zone3], server_ids=[2, 3, 4], host_ids=[2, 3, 4]}}, executor=10.10.10.39 2025-03-31 23:16:29.212 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:16:29.223 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obrs, tenantName=sys, dbUser=root 2025-03-31 23:16:29.225 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:16:29.228 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:16:29.230 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 23:16:29.232 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2025-03-31 23:16:29.244 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:16:29.246 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 23:16:29.248 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2025-03-31 23:16:29.262 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 23:16:29.264 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2025-03-31 23:16:29.267 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py on cluster, clusterId=2, obClusterId=1711351485, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm 2025-03-31 23:16:29.273 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 23:16:29.276 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 23:16:29.291 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py), params:null 2025-03-31 23:16:29.296 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:33447b6c-7231-43ea-b165-4550edcfa4b8 2025-03-31 23:16:29.301 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 23:16:29.304 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 23:16:29.320 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.5.2-102010012025022610_upgrade_health_checker_20250331231629.log, zone=null, cmdTimeout=0)), params:null 2025-03-31 23:16:29.356 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=33447b6c-7231-43ea-b165-4550edcfa4b8), params:null 2025-03-31 23:16:29.360 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2025-03-31 23:16:29.362 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2025-03-31 23:16:44.364 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=33447b6c-7231-43ea-b165-4550edcfa4b8), params:null 2025-03-31 23:16:44.369 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:true,null,, 2025-03-31 23:16:44.372 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,30f417712284adcc] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py finished Set state for subtask: 9171685, operation:RETRY, state: SUCCESSFUL Set state for subtask: 9171685, operation:RETRY, state: FAILED ############{RETRY}{2025-03-31T23:21:47.982+08:00}############ 2025-03-31 23:21:47.988 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.ocp.service.iam.user.UserService : user 1001002 login with organization 10000000 2025-03-31 23:21:47.991 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=9171685, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/data/log1, primary_root_sys_password=xxx task_instance_id=7941627, task_operation=retry, ob_run_path=/home/admin/oceanbase, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, obrs.1711351485.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"ten001\":\"True\",\"ten002\":\"True\",\"ten003\":\"True\",\"ten004\":\"True\",\"ten010\":\"True\",\"ten011\":\"True\",\"odc_meta\":\"True\",\"oms_meta\":\"True\",\"sys\":\"True\",\"ten005\":\"True\",\"ten006\":\"True\",\"ten007\":\"True\"}","enable_rereplication":"True"}, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade health checker script, watched_scenario=UPGRADE, sub_task_instance_id=9171685, primary_cluster_id=2, cluster_version=4.2.2.1, cluster_name=obrs, mix_rpm_names={"zone3":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1711351485, target_version=4.2.5.2-102010012025022610, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.5.2-102010012025022610.el7.x86_64.rpm","oceanbase-ce-utils-4.2.5.2-102010012025022610.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2025-03-31T23:21:47.977+08:00, ob_data_disk_path=/data/1}, listMap={standby_cluster_ids=[], 1711351485.obrs.zone3.server_ids=[4], 1711351485.o brs.zone1.server_ids=[2], cluster_ids=[2], primary_first_cluster_ids=[2], 1711351485.obrs.zone3.host_ids=[4], standby_first_cluster_ids=[2], 1711351485.obrs.zone2.host_ids=[3], staged_versions=[4.2.5.2-102010012025022610], 1711351485.obrs.zone2.server_ids=[3], direct_upgrade_versions=[], 1711351485.obrs.zone_names=[zone1, zone2, zone3], 1711351485.obrs.zone1.host_ids=[2], zone_names=[zone1, zone2, zone3], server_ids=[2, 3, 4], host_ids=[2, 3, 4]}}, executor=10.10.10.39 2025-03-31 23:21:48.010 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:21:48.022 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obrs, tenantName=sys, dbUser=root 2025-03-31 23:21:48.024 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:21:48.026 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:21:48.029 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 23:21:48.031 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2025-03-31 23:21:48.043 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obrs, obClusterId=1711351485, tenantName=sys, username=root, address=10.10.10.36, port=2881, database=oceanbase) 2025-03-31 23:21:48.046 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 23:21:48.048 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2025-03-31 23:21:48.059 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-31 23:21:48.062 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2025-03-31 23:21:48.066 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py on cluster, clusterId=2, obClusterId=1711351485, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm 2025-03-31 23:21:48.071 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 23:21:48.075 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 23:21:48.092 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py), params:null 2025-03-31 23:21:48.101 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:b68b2524-7f1a-43e6-9315-5ba622d0eaba 2025-03-31 23:21:48.106 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=4 2025-03-31 23:21:48.111 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 4 2025-03-31 23:21:48.127 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.5.2-102010012025022610.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_health_checker.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.5.2-102010012025022610_upgrade_health_checker_20250331232148.log, zone=null, cmdTimeout=0)), params:null 2025-03-31 23:21:48.161 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=b68b2524-7f1a-43e6-9315-5ba622d0eaba), params:null 2025-03-31 23:21:48.165 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2025-03-31 23:21:48.167 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2025-03-31 23:22:03.169 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.10.10.38:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=b68b2524-7f1a-43e6-9315-5ba622d0eaba), params:null 2025-03-31 23:22:03.175 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:true,null,, 2025-03-31 23:22:03.178 INFO 25034 --- [manual-subtask-executor12,a0327b7a330a68f8,94939848b1cd7083] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_health_checker.py finished Set state for subtask: 9171685, operation:RETRY, state: SUCCESSFUL Set state for subtask: 9171685, operation:RETRY, state: FAILED