2024-06-03 09:34:53.060 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.c.t.e.runner.JavaSubtaskRunner : Run subtask, id=19837485, context=Context{parallelIdx=0, stringMap={ob_log_disk_path=/data01/obredo, task_instance_id=13682619, task_operation=execute, ob_run_path=/home/admin/oceanbase, zone_name=zone2, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, cluster_id=4, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade post script, watched_scenario=UPGRADE, sub_task_instance_id=19837485, primary_cluster_id=4, cluster_version=4.2.1.0, obuat.3.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"interface_center\":\"True\",\"luban_global_id_aduit\":\"True\",\"cloud_object_storage\":\"True\",\"universe_search\":\"True\",\"business_audit_platform\":\"True\",\"vchat_obdb\":\"True\",\"serialcode_center_pt\":\"True\",\"sys\":\"True\",\"mq_gateway\":\"True\",\"sales\":\"True\",\"trade_push\":\"True\",\"uuc_login_aduit\":\"True\"}","enable_rereplication":"True"}, cluster_name=obuat, mix_rpm_names={"zone3":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=3, target_version=4.2.1.6-106000012024042515, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-06-03T09:34:52.897+08:00, ob_data_disk_path=/data01/obdata}, listMap={stan dby_cluster_ids=[], 3.obuat.zone_names=[zone3, zone1, zone2], 3.obuat.zone1.host_ids=[7], cluster_ids=[4], primary_first_cluster_ids=[4], 3.obuat.zone3.server_ids=[10], 3.obuat.zone1.server_ids=[8], zone_names=[zone3, zone1, zone2], 3.obuat.zone2.server_ids=[9], 3.obuat.zone3.host_ids=[5], server_ids=[9], standby_first_cluster_ids=[4], staged_versions=[4.2.1.6-106000012024042515], 3.obuat.zone2.host_ids=[6], host_ids=[6], direct_upgrade_versions=[]}}, executor=172.16.185.114 2024-06-03 09:34:53.069 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py on primary cluster, version=4.2.1.6-106000012024042515 2024-06-03 09:34:53.071 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py 2024-06-03 09:34:53.090 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 09:34:53.104 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obuat, tenantName=sys, dbUser=root 2024-06-03 09:34:53.108 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 09:34:53.112 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 09:34:53.115 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 09:34:53.118 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2024-06-03 09:34:53.135 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 09:34:53.137 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 09:34:53.140 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2024-06-03 09:34:53.150 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.165, sql port:2881 2024-06-03 09:34:53.153 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.165, sql port:2881 2024-06-03 09:34:53.155 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 09:34:53.159 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-06-03 09:34:53.165 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_post.py on cluster, clusterId=4, obClusterId=3, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm 2024-06-03 09:34:53.171 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=5 2024-06-03 09:34:53.174 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 5 2024-06-03 09:34:53.192 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py), params:null 2024-06-03 09:34:53.199 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:dc21556d-5e4a-4c0b-8279-77d8f4347335 2024-06-03 09:34:53.205 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=5 2024-06-03 09:34:53.209 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 5 2024-06-03 09:34:53.227 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.1.6-106000012024042515_upgrade_post_20240603093453.log, zone=null, cmdTimeout=0)), params:null 2024-06-03 09:34:53.236 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dc21556d-5e4a-4c0b-8279-77d8f4347335), params:null 2024-06-03 09:34:53.241 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2024-06-03 09:34:53.244 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2024-06-03 09:35:08.246 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dc21556d-5e4a-4c0b-8279-77d8f4347335), params:null 2024-06-03 09:35:08.266 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2024-06-03 09:35:08.269 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2024-06-03 09:35:23.271 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dc21556d-5e4a-4c0b-8279-77d8f4347335), params:null 2024-06-03 09:35:23.279 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2024-06-03 09:35:23.281 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2024-06-03 09:35:38.284 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dc21556d-5e4a-4c0b-8279-77d8f4347335), params:null 2024-06-03 09:35:38.292 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2024-06-03 09:35:38.295 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2024-06-03 09:35:53.297 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dc21556d-5e4a-4c0b-8279-77d8f4347335), params:null 2024-06-03 09:35:53.304 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2024-06-03 09:35:53.307 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2024-06-03 09:36:08.310 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dc21556d-5e4a-4c0b-8279-77d8f4347335), params:null 2024-06-03 09:36:08.317 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2024-06-03 09:36:08.320 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2024-06-03 09:36:23.323 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dc21556d-5e4a-4c0b-8279-77d8f4347335), params:null 2024-06-03 09:36:23.330 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2024-06-03 09:36:23.333 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2024-06-03 09:36:38.336 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dc21556d-5e4a-4c0b-8279-77d8f4347335), params:null 2024-06-03 09:36:38.344 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2024-06-03 09:36:38.348 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2024-06-03 09:36:53.351 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://172.16.103.165:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dc21556d-5e4a-4c0b-8279-77d8f4347335), params:null 2024-06-03 09:36:53.372 INFO 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,OcpAgentError: code = 1500, message = Execute shell command failed: failed to execute command: Command{user=root, program=sh, outputType=combined, cmd=export PYTHONPATH=/home/admin/ocp_agent/site-packages:$PYTHONPATH && python /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log, timeout=30m0s}, exitCode: 139, output: succeed to create run dir: /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE succeed to split all sub py files [2024-06-03 09:34:53] INFO do_upgrade_post.py:150 parameters from cmd: host="127.0.0.1", port=2881, user="root@sys", password=xxx timeout="0", module="set(['post_check', 'health_check', 'begin_upgrade', 'end_rolling_upgrade', 'begin_rolling_upgrade', 'end_upgrade', 'tenant_upgrade', 'special_action'])", log-file="/tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log" [2024-06-03 09:34:53] INFO actions.py:233 select distinct(substring_index(build_version, '_', 1)) from __all_server [2024-06-03 09:34:53] INFO actions.py:239 check server version success [2024-06-03 09:34:53] INFO do_upgrade_post.py:67 ================begin to run health check action =============== [2024-06-03 09:34:53] INFO upgrade_health_checker.py:288 zone is empty, check all servers in cluster [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):600, tenant_count:23, timeout_per_tenant(s):10, min_timeout(s):6 00 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status='inactive'), rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 0 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):600, tenant_count:23, timeout_per_tenant(s):10, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select count(*) from oceanbase.GV$OB_LOG_STAT where in_sync = 'NO', rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 0 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:336 check paxos replica success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):690, tenant_count:23, timeout_per_tenant(s):30, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b, rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 1 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:319 skip check server version by cluster [2024-06-03 09:34:53] INFO do_upgrade_post.py:69 ================succeed to run health check action =============== [2024-06-03 09:34:53] INFO do_upgrade_post.py:72 =========== =====begin to run end rolling upgrade action =============== [2024-06-03 09:34:53] INFO actions.py:245 select * from oceanbase.GV$OB_PARAMETERS where name = 'enable_upgrade_mode' and value = 'False' [2024-06-03 09:34:53] INFO actions.py:245 select * from oceanbase.GV$OB_PARAMETERS where name = '_upgrade_stage' and value = 'POSTUPGRADE' [2024-06-03 09:34:53] INFO actions.py:336 alter system end rolling upgrade [2024-06-03 09:34:53] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:53] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'min_observer_version' and value != '4.2.1.6' [2024-06-03 09:34:53] INFO actions.py:285 min_observer_version is not sync, value should be 4.2.1.6 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'min_observer_version' and value != '4.2.1.6' [2024-06-03 09:34:58] INFO actions.py:282 min_observer_version is sync, value is 4.2.1.6 [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = '_upgrade_stage' and value != 'POSTUPGRADE' [2024-06-03 09:34:58] INFO actions.py:282 _upgrade_stage is sync, value is POSTUPGRADE [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO do_upgrade_post.py:77 ================succeed to run end rolling upgrade action =============== [2024-06-03 09:34:58] INFO do_upgrade_post.py:80 ================begin to run tenant upgrade action =============== [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select count(*) from oceanbase.__all_table where table_name = '__all_virtual_core_table' [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select tenant_id from oceanbase.__all_tenant [2024-06-03 09:34:5 8] INFO tenant_upgrade_action.py:47 select count(*) from __all_virtual_core_table where column_name in ('target_data_version', 'current_data_version') and column_value = 17180000518 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:96 target_data_version/current_data_version not match with 4.2.1.6, tenant_cnt:23, result_cnt:0 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:232 start to run upgrade job, job_name:UPGRADE_ALL [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select job_status from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' order by job_id desc limit 1 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:156 maybe upgrade job remained, can run again [2024-06-03 09:34:58] INFO actions.py:183 select value from oceanbase.__all_sys_parameter where name = 'enable_ddl' [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'False' [2024-06-03 09:34:58] INFO actions.py:282 enable_ddl is sync, value is False [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO actions.py:126 alter system set enable_ddl = 'True' [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'True' [2024-06-03 09:34:58] INFO actions.py:285 enable_ddl is not sync, value should be True [2024-06-03 09:35:03] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'True' [2024-06-03 09:35:03] INFO actions.py:282 enable_ddl is sync, value is True [2024-06-03 09:35:03] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:35:03] INFO actions.py:126 alter system set enable_sys_table_ddl = 'True' [2024-06-03 09:35:03] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:35:03] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_sys_table_ddl' and value != 'True' [2024-06-03 09:35:03] INFO actions.py:285 enable_sys_table_ddl is not sync, value should be True [2024-06-03 09:35:08] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_sys_table_ddl' and value != 'True' [2024-06-03 09:35:08] INFO actions.py:282 enable_sys_table_ddl is sync, value is True [2024-06-03 09:35:08] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:47 select job_id from oceanbase.__all_rootservice_job order by job_id desc limit 1 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:131 get max_used_job_id:2 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:246 alter system run upgrade job 'UPGRADE_ALL' [2024-06-03 09:35:08] INFO actions.py:421 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:35:08] INFO actions.py:146 use default timeout caculated by tenants, timeout(s):3600, tenant_count:23, timeout_per_tenant(s):100, min_timeout(s):3600 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:181 upgrade job not created yet [2024-06-03 09:35:18] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:18] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:28] INFO tenant_upgrade_action.py:47 select job_status, r s_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:28] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:38] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:38] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:49] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:49] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:59] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:59] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:09] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:09] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:19] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:19] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:29] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from ocean base.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:29] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:39] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:215 execute upgrade job failed [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:227 failed to check upgrade job result [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:256 run upgrade job failed, :UPGRADE_ALL [2024-06-03 09:36:39] ERROR do_upgrade_post.py:104 run error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 82, in do_upgrade tenant_upgrade_action.do_upgrade(conn, cur, timeout, my_user, my_passwd) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/tenant_upgrade_action.py", line 20, in do_upgrade run_upgrade_job(conn, cur, "UPGRADE_ALL", timeout) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/tenant_upgrade_action.py", line 257, in run_upgrade_job raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] INFO do_upgrade_post.py:43 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:44 ============================== STATISTICS BEGIN ================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:45 =============================== =================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:46 succeed run sql(except sql of special actions): alter system end rolling upgrade; [2024-06-03 09:36:39] INFO do_upgrade_post.py:47 commited sql(except sql of special actions): alter system end rolling upgrade; [2024-06-03 09:36:39] INFO do_upgrade_post.py:48 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:49 =============================== STATISTICS END =================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:50 ================================================================================== [2024-06-03 09:36:39] ERROR do_upgrade_post.py:117 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 105, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] ERROR do_upgrade_post.py:157 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 151, in do_upgrade_by_argv do_upgrade(host, port, user, password, timeout, module_set, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 118, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] ERROR do_upgrade_post.py:158 run error, maybe you can reference rollback_sql_post.txt to rollback it Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin /oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 151, in do_upgrade_by_argv do_upgrade(host, port, user, password, timeout, module_set, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 118, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment sh: line 1: 97420 Segmentation fault (core dumped) python /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log , error: exit status 139, 2024-06-03 09:36:53.397 ERROR 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.s.task.util.AgentAsyncTaskHelper : failed to execute async task, task token: dc21556d-5e4a-4c0b-8279-77d8f4347335, err: OcpAgentError: code = 1500, message = Execute shell command failed: failed to execute command: Command{user=root, program=sh, outputType=combined, cmd=export PYTHONPATH=/home/admin/ocp_agent/site-packages:$PYTHONPATH && python /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log, timeout=30m0s}, exitCode: 139, output: succeed to create run dir: /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE succeed to split all sub py files [2024-06-03 09:34:53] INFO do_upgrade_post.py:150 parameters from cmd: host="127.0.0.1", port=2881, user="root@sys", password=xxx timeout="0", module="set(['post_check', 'health_check', 'begin_upgrade', 'end_rolling_upgrade', 'begin_rolling_upgrade', 'end_upgrade', 'tenant_upgrade', 'special_action'])", log-file="/tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log" [2024-06-03 09:34:53] INFO actions.py:233 select distinct(substring_index(build_version, '_', 1)) from __all_server [2024-06-03 09:34:53] INFO actions.py:239 check server version success [2024-06-03 09:34:53] INFO do_upgrade_post.py:67 ================begin to run health check action =============== [2024-06-03 09:34:53] INFO upgrade_health_checker.py:288 zone is empty, check all servers in cluster [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):600, tenant_count:23, timeout_per_ tenant(s):10, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status='inactive'), rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 0 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):600, tenant_count:23, timeout_per_tenant(s):10, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select count(*) from oceanbase.GV$OB_LOG_STAT where in_sync = 'NO', rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 0 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:336 check paxos replica success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):690, tenant_count:23, timeout_per_tenant(s):30, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b, rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 1 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:319 skip check server version by cluster [2024-06-03 09:34:53] INFO do_upgrade_post.py:69 ================succeed to run health check action =============== [2024-06-03 09:34:53] INFO do_ upgrade_post.py:72 ================begin to run end rolling upgrade action =============== [2024-06-03 09:34:53] INFO actions.py:245 select * from oceanbase.GV$OB_PARAMETERS where name = 'enable_upgrade_mode' and value = 'False' [2024-06-03 09:34:53] INFO actions.py:245 select * from oceanbase.GV$OB_PARAMETERS where name = '_upgrade_stage' and value = 'POSTUPGRADE' [2024-06-03 09:34:53] INFO actions.py:336 alter system end rolling upgrade [2024-06-03 09:34:53] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:53] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'min_observer_version' and value != '4.2.1.6' [2024-06-03 09:34:53] INFO actions.py:285 min_observer_version is not sync, value should be 4.2.1.6 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'min_observer_version' and value != '4.2.1.6' [2024-06-03 09:34:58] INFO actions.py:282 min_observer_version is sync, value is 4.2.1.6 [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = '_upgrade_stage' and value != 'POSTUPGRADE' [2024-06-03 09:34:58] INFO actions.py:282 _upgrade_stage is sync, value is POSTUPGRADE [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO do_upgrade_post.py:77 ================succeed to run end rolling upgrade action =============== [2024-06-03 09:34:58] INFO do_upgrade_post.py:80 ================begin to run tenant upgrade action =============== [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select count(*) from oceanbase.__all_table where table_name = '__all_virtual_core_table' [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select tenant_id from oceanbase.__ all_tenant [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select count(*) from __all_virtual_core_table where column_name in ('target_data_version', 'current_data_version') and column_value = 17180000518 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:96 target_data_version/current_data_version not match with 4.2.1.6, tenant_cnt:23, result_cnt:0 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:232 start to run upgrade job, job_name:UPGRADE_ALL [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select job_status from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' order by job_id desc limit 1 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:156 maybe upgrade job remained, can run again [2024-06-03 09:34:58] INFO actions.py:183 select value from oceanbase.__all_sys_parameter where name = 'enable_ddl' [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'False' [2024-06-03 09:34:58] INFO actions.py:282 enable_ddl is sync, value is False [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO actions.py:126 alter system set enable_ddl = 'True' [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'True' [2024-06-03 09:34:58] INFO actions.py:285 enable_ddl is not sync, value should be True [2024-06-03 09:35:03] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'True' [2024-06-03 09:35:03] INFO actions.py:282 enable_ddl is sync, value is True [2024-06-03 09:35:03] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:35:03] INFO actions.py:126 alter system set enable_sys_ table_ddl = 'True' [2024-06-03 09:35:03] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:35:03] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_sys_table_ddl' and value != 'True' [2024-06-03 09:35:03] INFO actions.py:285 enable_sys_table_ddl is not sync, value should be True [2024-06-03 09:35:08] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_sys_table_ddl' and value != 'True' [2024-06-03 09:35:08] INFO actions.py:282 enable_sys_table_ddl is sync, value is True [2024-06-03 09:35:08] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:47 select job_id from oceanbase.__all_rootservice_job order by job_id desc limit 1 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:131 get max_used_job_id:2 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:246 alter system run upgrade job 'UPGRADE_ALL' [2024-06-03 09:35:08] INFO actions.py:421 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:35:08] INFO actions.py:146 use default timeout caculated by tenants, timeout(s):3600, tenant_count:23, timeout_per_tenant(s):100, min_timeout(s):3600 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:181 upgrade job not created yet [2024-06-03 09:35:18] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:18] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:28] INFO tenant_upgrade_act ion.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:28] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:38] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:38] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:49] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:49] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:59] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:59] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:09] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:09] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:19] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:19] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:29] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_s vr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:29] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:39] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:215 execute upgrade job failed [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:227 failed to check upgrade job result [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:256 run upgrade job failed, :UPGRADE_ALL [2024-06-03 09:36:39] ERROR do_upgrade_post.py:104 run error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 82, in do_upgrade tenant_upgrade_action.do_upgrade(conn, cur, timeout, my_user, my_passwd) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/tenant_upgrade_action.py", line 20, in do_upgrade run_upgrade_job(conn, cur, "UPGRADE_ALL", timeout) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/tenant_upgrade_action.py", line 257, in run_upgrade_job raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] INFO do_upgrade_post.py:43 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:44 ============================== STATISTICS BEGIN ================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:45 = ================================================================================= [2024-06-03 09:36:39] INFO do_upgrade_post.py:46 succeed run sql(except sql of special actions): alter system end rolling upgrade; [2024-06-03 09:36:39] INFO do_upgrade_post.py:47 commited sql(except sql of special actions): alter system end rolling upgrade; [2024-06-03 09:36:39] INFO do_upgrade_post.py:48 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:49 =============================== STATISTICS END =================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:50 ================================================================================== [2024-06-03 09:36:39] ERROR do_upgrade_post.py:117 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 105, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] ERROR do_upgrade_post.py:157 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 151, in do_upgrade_by_argv do_upgrade(host, port, user, password, timeout, module_set, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 118, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] ERROR do_upgrade_post.py:158 run error, maybe you can reference rollback_sql_post.txt to rollback it Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-10600001202404 2515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 151, in do_upgrade_by_argv do_upgrade(host, port, user, password, timeout, module_set, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 118, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment sh: line 1: 97420 Segmentation fault (core dumped) python /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log , error: exit status 139 2024-06-03 09:36:53.421 ERROR 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.ocp.core.util.ExceptionUtils : Checked Exception: com.oceanbase.ocp.core.exception.UnexpectedException occurred with code error.agent.task.failed, and args [dc21556d-5e4a-4c0b-8279-77d8f4347335, OcpAgentError: code = 1500, message = Execute shell command failed: failed to execute command: Command{user=root, program=sh, outputType=combined, cmd=export PYTHONPATH=/home/admin/ocp_agent/site-packages:$PYTHONPATH && python /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log, timeout=30m0s}, exitCode: 139, output: succeed to create run dir: /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE succeed to split all sub py files [2024-06-03 09:34:53] INFO do_upgrade_post.py:150 parameters from cmd: host="127.0.0.1", port=2881, user="root@sys", password=xxx timeout="0", module="set(['post_check', 'health_check', 'begin_upgrade', 'end_rolling_upgrade', 'begin_rolling_upgrade', 'end_upgrade', 'tenant_upgrade', 'special_action'])", log-file="/tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log" [2024-06-03 09:34:53] INFO actions.py:233 select distinct(substring_index(build_version, '_', 1)) from __all_server [2024-06-03 09:34:53] INFO actions.py:239 check server version success [2024-06-03 09:34:53] INFO do_upgrade_post.py:67 ================begin to run health check action =============== [2024-06-03 09:34:53] INFO upgrade_health_checker.py:288 zone is empty, check all servers in cluster [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use defa ult timeout caculated by tenants, timeout(s):600, tenant_count:23, timeout_per_tenant(s):10, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status='inactive'), rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 0 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):600, tenant_count:23, timeout_per_tenant(s):10, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select count(*) from oceanbase.GV$OB_LOG_STAT where in_sync = 'NO', rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 0 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:336 check paxos replica success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):690, tenant_count:23, timeout_per_tenant(s):30, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b, rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 1 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:319 skip check server version by cluster [2024-06-03 09:34:53] INFO do_upgrade_post.py:69 ================su cceed to run health check action =============== [2024-06-03 09:34:53] INFO do_upgrade_post.py:72 ================begin to run end rolling upgrade action =============== [2024-06-03 09:34:53] INFO actions.py:245 select * from oceanbase.GV$OB_PARAMETERS where name = 'enable_upgrade_mode' and value = 'False' [2024-06-03 09:34:53] INFO actions.py:245 select * from oceanbase.GV$OB_PARAMETERS where name = '_upgrade_stage' and value = 'POSTUPGRADE' [2024-06-03 09:34:53] INFO actions.py:336 alter system end rolling upgrade [2024-06-03 09:34:53] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:53] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'min_observer_version' and value != '4.2.1.6' [2024-06-03 09:34:53] INFO actions.py:285 min_observer_version is not sync, value should be 4.2.1.6 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'min_observer_version' and value != '4.2.1.6' [2024-06-03 09:34:58] INFO actions.py:282 min_observer_version is sync, value is 4.2.1.6 [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = '_upgrade_stage' and value != 'POSTUPGRADE' [2024-06-03 09:34:58] INFO actions.py:282 _upgrade_stage is sync, value is POSTUPGRADE [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO do_upgrade_post.py:77 ================succeed to run end rolling upgrade action =============== [2024-06-03 09:34:58] INFO do_upgrade_post.py:80 ================begin to run tenant upgrade action =============== [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select count(*) from oceanbase.__all_table where table_name = '__all_virtual_core_table' [2024-06-0 3 09:34:58] INFO tenant_upgrade_action.py:47 select tenant_id from oceanbase.__all_tenant [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select count(*) from __all_virtual_core_table where column_name in ('target_data_version', 'current_data_version') and column_value = 17180000518 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:96 target_data_version/current_data_version not match with 4.2.1.6, tenant_cnt:23, result_cnt:0 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:232 start to run upgrade job, job_name:UPGRADE_ALL [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select job_status from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' order by job_id desc limit 1 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:156 maybe upgrade job remained, can run again [2024-06-03 09:34:58] INFO actions.py:183 select value from oceanbase.__all_sys_parameter where name = 'enable_ddl' [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'False' [2024-06-03 09:34:58] INFO actions.py:282 enable_ddl is sync, value is False [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO actions.py:126 alter system set enable_ddl = 'True' [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'True' [2024-06-03 09:34:58] INFO actions.py:285 enable_ddl is not sync, value should be True [2024-06-03 09:35:03] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'True' [2024-06-03 09:35:03] INFO actions.py:282 enable_ddl is sync, value is True [2024-06-03 09:35:03] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:35:03] INFO actions.py:126 alter system set enable_sys_table_ddl = 'True' [2024-06-03 09:35:03] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:35:03] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_sys_table_ddl' and value != 'True' [2024-06-03 09:35:03] INFO actions.py:285 enable_sys_table_ddl is not sync, value should be True [2024-06-03 09:35:08] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_sys_table_ddl' and value != 'True' [2024-06-03 09:35:08] INFO actions.py:282 enable_sys_table_ddl is sync, value is True [2024-06-03 09:35:08] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:47 select job_id from oceanbase.__all_rootservice_job order by job_id desc limit 1 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:131 get max_used_job_id:2 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:246 alter system run upgrade job 'UPGRADE_ALL' [2024-06-03 09:35:08] INFO actions.py:421 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:35:08] INFO actions.py:146 use default timeout caculated by tenants, timeout(s):3600, tenant_count:23, timeout_per_tenant(s):100, min_timeout(s):3600 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:181 upgrade job not created yet [2024-06-03 09:35:18] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:18] INFO tenant_upgrade_action.py :186 upgrade job is still running [2024-06-03 09:35:28] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:28] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:38] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:38] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:49] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:49] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:59] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:59] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:09] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:09] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:19] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:19] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-0 3 09:36:29] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:29] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:39] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:215 execute upgrade job failed [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:227 failed to check upgrade job result [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:256 run upgrade job failed, :UPGRADE_ALL [2024-06-03 09:36:39] ERROR do_upgrade_post.py:104 run error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 82, in do_upgrade tenant_upgrade_action.do_upgrade(conn, cur, timeout, my_user, my_passwd) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/tenant_upgrade_action.py", line 20, in do_upgrade run_upgrade_job(conn, cur, "UPGRADE_ALL", timeout) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/tenant_upgrade_action.py", line 257, in run_upgrade_job raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] INFO do_upgrade_post.py:43 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:44 ============================== STATISTICS BEGIN ====== ============================ [2024-06-03 09:36:39] INFO do_upgrade_post.py:45 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:46 succeed run sql(except sql of special actions): alter system end rolling upgrade; [2024-06-03 09:36:39] INFO do_upgrade_post.py:47 commited sql(except sql of special actions): alter system end rolling upgrade; [2024-06-03 09:36:39] INFO do_upgrade_post.py:48 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:49 =============================== STATISTICS END =================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:50 ================================================================================== [2024-06-03 09:36:39] ERROR do_upgrade_post.py:117 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 105, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] ERROR do_upgrade_post.py:157 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 151, in do_upgrade_by_argv do_upgrade(host, port, user, password, timeout, module_set, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 118, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] ERROR do_upgrade_post.py:158 run error, maybe you can reference rollback_sql_post.txt to rollback it Traceback (most r ecent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 151, in do_upgrade_by_argv do_upgrade(host, port, user, password, timeout, module_set, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 118, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment sh: line 1: 97420 Segmentation fault (core dumped) python /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log , error: exit status 139] 2024-06-03 09:36:53.442 ERROR 54383 --- [pool-manual-subtask-executor13,2aa5e462d52e4eca,092f53ad358a] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : OCP-Agent task dc21556d-5e4a-4c0b-8279-77d8f4347335 failed, error message: OcpAgentError: code = 1500, message = Execute shell command failed: failed to execute command: Command{user=root, program=sh, outputType=combined, cmd=export PYTHONPATH=/home/admin/ocp_agent/site-packages:$PYTHONPATH && python /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log, timeout=30m0s}, exitCode: 139, output: succeed to create run dir: /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE succeed to split all sub py files [2024-06-03 09:34:53] INFO do_upgrade_post.py:150 parameters from cmd: host="127.0.0.1", port=2881, user="root@sys", password=xxx timeout="0", module="set(['post_check', 'health_check', 'begin_upgrade', 'end_rolling_upgrade', 'begin_rolling_upgrade', 'end_upgrade', 'tenant_upgrade', 'special_action'])", log-file="/tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log" [2024-06-03 09:34:53] INFO actions.py:233 select distinct(substring_index(build_version, '_', 1)) from __all_server [2024-06-03 09:34:53] INFO actions.py:239 check server version success [2024-06-03 09:34:53] INFO do_upgrade_post.py:67 ================begin to run health check action =============== [2024-06-03 09:34:53] INFO upgrade_health_checker.py:288 zone is empty, check all servers in cluster [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):600, tenant_count:23, timeout_per_tenant(s): 10, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status='inactive'), rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 0 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):600, tenant_count:23, timeout_per_tenant(s):10, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select count(*) from oceanbase.GV$OB_LOG_STAT where in_sync = 'NO', rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 0 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:336 check paxos replica success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):690, tenant_count:23, timeout_per_tenant(s):30, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b, rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 1 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:319 skip check server version by cluster [2024-06-03 09:34:53] INFO do_upgrade_post.py:69 ================succeed to run health check action =============== [2024-06-03 09:34:53] INFO do_upgrade_po st.py:72 ================begin to run end rolling upgrade action =============== [2024-06-03 09:34:53] INFO actions.py:245 select * from oceanbase.GV$OB_PARAMETERS where name = 'enable_upgrade_mode' and value = 'False' [2024-06-03 09:34:53] INFO actions.py:245 select * from oceanbase.GV$OB_PARAMETERS where name = '_upgrade_stage' and value = 'POSTUPGRADE' [2024-06-03 09:34:53] INFO actions.py:336 alter system end rolling upgrade [2024-06-03 09:34:53] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:53] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'min_observer_version' and value != '4.2.1.6' [2024-06-03 09:34:53] INFO actions.py:285 min_observer_version is not sync, value should be 4.2.1.6 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'min_observer_version' and value != '4.2.1.6' [2024-06-03 09:34:58] INFO actions.py:282 min_observer_version is sync, value is 4.2.1.6 [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = '_upgrade_stage' and value != 'POSTUPGRADE' [2024-06-03 09:34:58] INFO actions.py:282 _upgrade_stage is sync, value is POSTUPGRADE [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO do_upgrade_post.py:77 ================succeed to run end rolling upgrade action =============== [2024-06-03 09:34:58] INFO do_upgrade_post.py:80 ================begin to run tenant upgrade action =============== [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select count(*) from oceanbase.__all_table where table_name = '__all_virtual_core_table' [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select tenant_id from oceanbase.__all_tenant [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select count(*) from __all_virtual_core_table where column_name in ('target_data_version', 'current_data_version') and column_value = 17180000518 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:96 target_data_version/current_data_version not match with 4.2.1.6, tenant_cnt:23, result_cnt:0 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:232 start to run upgrade job, job_name:UPGRADE_ALL [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select job_status from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' order by job_id desc limit 1 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:156 maybe upgrade job remained, can run again [2024-06-03 09:34:58] INFO actions.py:183 select value from oceanbase.__all_sys_parameter where name = 'enable_ddl' [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'False' [2024-06-03 09:34:58] INFO actions.py:282 enable_ddl is sync, value is False [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO actions.py:126 alter system set enable_ddl = 'True' [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'True' [2024-06-03 09:34:58] INFO actions.py:285 enable_ddl is not sync, value should be True [2024-06-03 09:35:03] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'True' [2024-06-03 09:35:03] INFO actions.py:282 enable_ddl is sync, value is True [2024-06-03 09:35:03] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:35:03] INFO actions.py:126 alter system set enable_sys_table_ddl = 'True' [2024-06-03 09:35:03] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:35:03] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_sys_table_ddl' and value != 'True' [2024-06-03 09:35:03] INFO actions.py:285 enable_sys_table_ddl is not sync, value should be True [2024-06-03 09:35:08] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_sys_table_ddl' and value != 'True' [2024-06-03 09:35:08] INFO actions.py:282 enable_sys_table_ddl is sync, value is True [2024-06-03 09:35:08] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:47 select job_id from oceanbase.__all_rootservice_job order by job_id desc limit 1 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:131 get max_used_job_id:2 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:246 alter system run upgrade job 'UPGRADE_ALL' [2024-06-03 09:35:08] INFO actions.py:421 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:35:08] INFO actions.py:146 use default timeout caculated by tenants, timeout(s):3600, tenant_count:23, timeout_per_tenant(s):100, min_timeout(s):3600 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:181 upgrade job not created yet [2024-06-03 09:35:18] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:18] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:28] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:28] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:38] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:38] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:49] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:49] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:59] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:59] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:09] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:09] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:19] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:19] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:29] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, g mt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:29] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:39] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:215 execute upgrade job failed [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:227 failed to check upgrade job result [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:256 run upgrade job failed, :UPGRADE_ALL [2024-06-03 09:36:39] ERROR do_upgrade_post.py:104 run error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 82, in do_upgrade tenant_upgrade_action.do_upgrade(conn, cur, timeout, my_user, my_passwd) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/tenant_upgrade_action.py", line 20, in do_upgrade run_upgrade_job(conn, cur, "UPGRADE_ALL", timeout) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/tenant_upgrade_action.py", line 257, in run_upgrade_job raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] INFO do_upgrade_post.py:43 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:44 ============================== STATISTICS BEGIN ================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:45 =========== ======================================================================= [2024-06-03 09:36:39] INFO do_upgrade_post.py:46 succeed run sql(except sql of special actions): alter system end rolling upgrade; [2024-06-03 09:36:39] INFO do_upgrade_post.py:47 commited sql(except sql of special actions): alter system end rolling upgrade; [2024-06-03 09:36:39] INFO do_upgrade_post.py:48 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:49 =============================== STATISTICS END =================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:50 ================================================================================== [2024-06-03 09:36:39] ERROR do_upgrade_post.py:117 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 105, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] ERROR do_upgrade_post.py:157 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 151, in do_upgrade_by_argv do_upgrade(host, port, user, password, timeout, module_set, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 118, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] ERROR do_upgrade_post.py:158 run error, maybe you can reference rollback_sql_post.txt to rollback it Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x 86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 151, in do_upgrade_by_argv do_upgrade(host, port, user, password, timeout, module_set, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 118, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment sh: line 1: 97420 Segmentation fault (core dumped) python /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log , error: exit status 139 com.oceanbase.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=AGENT_TASK_FAILED, args=dc21556d-5e4a-4c0b-8279-77d8f4347335,OcpAgentError: code = 1500, message = Execute shell command failed: failed to execute command: Command{user=root, program=sh, outputType=combined, cmd=export PYTHONPATH=/home/admin/ocp_agent/site-packages:$PYTHONPATH && python /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log, timeout=30m0s}, exitCode: 139, output: succeed to create run dir: /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE succeed to split all sub py files [2024-06-03 09:34:53] INFO do_upgrade_post.py:150 parameters from cmd: host="127.0.0.1", port=2881, user="root@sys", password=xxx timeout="0", module="set(['post_check', 'health_check', 'begin_upgrade', 'end_rolling_upgrade', 'begin_rolling_upgrade', 'end_upgrade', 'tenant_upgrade', 'special_action'])", lo g-file="/tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log" [2024-06-03 09:34:53] INFO actions.py:233 select distinct(substring_index(build_version, '_', 1)) from __all_server [2024-06-03 09:34:53] INFO actions.py:239 check server version success [2024-06-03 09:34:53] INFO do_upgrade_post.py:67 ================begin to run health check action =============== [2024-06-03 09:34:53] INFO upgrade_health_checker.py:288 zone is empty, check all servers in cluster [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):600, tenant_count:23, timeout_per_tenant(s):10, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status='inactive'), rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 0 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):600, tenant_count:23, timeout_per_tenant(s):10, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select count(*) from oceanbase.GV$OB_LOG_STAT where in_sync = 'NO', rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 0 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:336 check paxos replica success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:311 use default timeout caculated by tenants, timeout(s):690, tenant_count:23, timeout_per_tenant(s):30, min_timeout(s):600 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:46 succeed to execute query: select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b, rowcount = 1 [2024-06-03 09:34:53] INFO upgrade_health_checker.py:375 check value is 1 success [2024-06-03 09:34:53] INFO upgrade_health_checker.py:319 skip check server version by cluster [2024-06-03 09:34:53] INFO do_upgrade_post.py:69 ================succeed to run health check action =============== [2024-06-03 09:34:53] INFO do_upgrade_post.py:72 ================begin to run end rolling upgrade action =============== [2024-06-03 09:34:53] INFO actions.py:245 select * from oceanbase.GV$OB_PARAMETERS where name = 'enable_upgrade_mode' and value = 'False' [2024-06-03 09:34:53] INFO actions.py:245 select * from oceanbase.GV$OB_PARAMETERS where name = '_upgrade_stage' and value = 'POSTUPGRADE' [2024-06-03 09:34:53] INFO actions.py:336 alter system end rolling upgrade [2024-06-03 09:34:53] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:53] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'min_observer_version' and value != '4.2.1.6' [2024-06-03 09:34:53] INFO actions.py:285 min_observer_version is not sync, value should be 4.2.1.6 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'min_observer_version' and value != '4.2.1.6' [2024-06-03 09:34:58] INFO actions.py:282 min_observer_version is sync, value is 4.2.1.6 [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09 :34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = '_upgrade_stage' and value != 'POSTUPGRADE' [2024-06-03 09:34:58] INFO actions.py:282 _upgrade_stage is sync, value is POSTUPGRADE [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:34:58] INFO do_upgrade_post.py:77 ================succeed to run end rolling upgrade action =============== [2024-06-03 09:34:58] INFO do_upgrade_post.py:80 ================begin to run tenant upgrade action =============== [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select count(*) from oceanbase.__all_table where table_name = '__all_virtual_core_table' [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select tenant_id from oceanbase.__all_tenant [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select count(*) from __all_virtual_core_table where column_name in ('target_data_version', 'current_data_version') and column_value = 17180000518 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:96 target_data_version/current_data_version not match with 4.2.1.6, tenant_cnt:23, result_cnt:0 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:232 start to run upgrade job, job_name:UPGRADE_ALL [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:47 select job_status from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' order by job_id desc limit 1 [2024-06-03 09:34:58] INFO tenant_upgrade_action.py:156 maybe upgrade job remained, can run again [2024-06-03 09:34:58] INFO actions.py:183 select value from oceanbase.__all_sys_parameter where name = 'enable_ddl' [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'False' [2024-06-03 09:34:58] INFO actions.py:282 enable_ddl is sync, value is False [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 100 00000 [2024-06-03 09:34:58] INFO actions.py:126 alter system set enable_ddl = 'True' [2024-06-03 09:34:58] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:34:58] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'True' [2024-06-03 09:34:58] INFO actions.py:285 enable_ddl is not sync, value should be True [2024-06-03 09:35:03] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_ddl' and value != 'True' [2024-06-03 09:35:03] INFO actions.py:282 enable_ddl is sync, value is True [2024-06-03 09:35:03] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:35:03] INFO actions.py:126 alter system set enable_sys_table_ddl = 'True' [2024-06-03 09:35:03] INFO actions.py:132 set @@session.ob_query_timeout = 60000000 [2024-06-03 09:35:03] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_sys_table_ddl' and value != 'True' [2024-06-03 09:35:03] INFO actions.py:285 enable_sys_table_ddl is not sync, value should be True [2024-06-03 09:35:08] INFO actions.py:275 select count(*) as cnt from oceanbase.GV$OB_PARAMETERS where name = 'enable_sys_table_ddl' and value != 'True' [2024-06-03 09:35:08] INFO actions.py:282 enable_sys_table_ddl is sync, value is True [2024-06-03 09:35:08] INFO actions.py:132 set @@session.ob_query_timeout = 10000000 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:47 select job_id from oceanbase.__all_rootservice_job order by job_id desc limit 1 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:131 get max_used_job_id:2 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:246 alter system run upgrade job 'UPGRADE_ALL' [2024-06-03 09:35:08] INFO actions.py:421 succeed to execute query: select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc, rowcount = 23 [2024-06-03 09:35:08] INFO actions.py:146 use default timeout caculated by tenants, timeout(s):3600, tenant_count:23, timeout_per_tenant(s):100, min_timeout(s):3600 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:08] INFO tenant_upgrade_action.py:181 upgrade job not created yet [2024-06-03 09:35:18] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:18] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:28] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:28] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:38] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:38] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:49] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:49] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:35:59] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:35:59] INFO tenant_upgrade_action .py:186 upgrade job is still running [2024-06-03 09:36:09] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:09] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:19] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:19] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:29] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:29] INFO tenant_upgrade_action.py:186 upgrade job is still running [2024-06-03 09:36:39] INFO tenant_upgrade_action.py:47 select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = 'UPGRADE_ALL' and job_id > 2 order by job_id desc limit 1 [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:215 execute upgrade job failed [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:227 failed to check upgrade job result [2024-06-03 09:36:39] WARNING tenant_upgrade_action.py:256 run upgrade job failed, :UPGRADE_ALL [2024-06-03 09:36:39] ERROR do_upgrade_post.py:104 run error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 82, in do_upgrade tenant_upgrade_action.do_upgrade(conn, cur, timeout, my_user, my_passwd) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanb ase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/tenant_upgrade_action.py", line 20, in do_upgrade run_upgrade_job(conn, cur, "UPGRADE_ALL", timeout) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/tenant_upgrade_action.py", line 257, in run_upgrade_job raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] INFO do_upgrade_post.py:43 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:44 ============================== STATISTICS BEGIN ================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:45 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:46 succeed run sql(except sql of special actions): alter system end rolling upgrade; [2024-06-03 09:36:39] INFO do_upgrade_post.py:47 commited sql(except sql of special actions): alter system end rolling upgrade; [2024-06-03 09:36:39] INFO do_upgrade_post.py:48 ================================================================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:49 =============================== STATISTICS END =================================== [2024-06-03 09:36:39] INFO do_upgrade_post.py:50 ================================================================================== [2024-06-03 09:36:39] ERROR do_upgrade_post.py:117 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 105, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] ERROR do_upgrade_post.py:157 normal error Traceback (most recent call last): File "/tmp/rpms/ex tract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 151, in do_upgrade_by_argv do_upgrade(host, port, user, password, timeout, module_set, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 118, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment [2024-06-03 09:36:39] ERROR do_upgrade_post.py:158 run error, maybe you can reference rollback_sql_post.txt to rollback it Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 151, in do_upgrade_by_argv do_upgrade(host, port, user, password, timeout, module_set, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post_extract_files_2024_06_03_09_34_53_262162_MkqHsXSE/do_upgrade_post.py", line 118, in do_upgrade raise e UnboundLocalError: local variable 'e' referenced before assignment sh: line 1: 97420 Segmentation fault (core dumped) python /tmp/rpms/extract/oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_post.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.6-106000012024042515_upgrade_post_20240603093453.log , error: exit status 139 at com.oceanbase.ocp.core.i18n.ErrorCodes$Kind$10.exception(ErrorCodes.java:1570) at com.oceanbase.ocp.core.i18n.ErrorCodes.exception(ErrorCodes.java:1425) at com.oceanbase.ocp.core.util.ExceptionUtils.newException(ExceptionUtils.java:169) at com.oceanbase.ocp.core.util.ExceptionUtils.throwException(ExceptionUtils.java:162) at com.oceanbase.ocp.service.task.util.AgentAsyncTaskHelper.checkSuccess(Ag entAsyncTaskHelper.java:229) at com.oceanbase.ocp.service.task.util.AgentAsyncTaskHelper.waitForExecuteFinish(AgentAsyncTaskHelper.java:175) at com.oceanbase.ocp.service.task.util.AgentAsyncTaskHelper.runUpgradeScript(AgentAsyncTaskHelper.java:130) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:160) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:105) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:80) at com.oceanbase.ocp.service.task.business.cluster.ExecUpgradePostScriptTask.run(ExecUpgradePostScriptTask.java:66) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.execute(JavaSubtaskRunner.java:64) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:32) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 19837485, operation:EXECUTE, state: FAILED Set state for subtask: 19837485, operation:EXECUTE, state: FAILED 2024-06-03 10:35:04.286 WARN 54383 --- [pool-subtask-executor2319,917c8ec7bbc545a7,04838081a817] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Subtask timeout, name=Execute upgrade post script, executedSeconds=3611, timeout=3600 2024-06-03 11:51:49.018 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=19837485, context=Context{parallelIdx=0, stringMap={ob_log_disk_path=/data01/obredo, task_instance_id=13682619, task_operation=retry, ob_run_path=/home/admin/oceanbase, zone_name=zone2, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, cluster_id=4, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade post script, watched_scenario=UPGRADE, sub_task_instance_id=19837485, primary_cluster_id=4, cluster_version=4.2.1.0, obuat.3.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"interface_center\":\"True\",\"luban_global_id_aduit\":\"True\",\"cloud_object_storage\":\"True\",\"universe_search\":\"True\",\"business_audit_platform\":\"True\",\"vchat_obdb\":\"True\",\"serialcode_center_pt\":\"True\",\"sys\":\"True\",\"mq_gateway\":\"True\",\"sales\":\"True\",\"trade_push\":\"True\",\"uuc_login_aduit\":\"True\"}","enable_rereplication":"True"}, cluster_name=obuat, mix_rpm_names={"zone3":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=3, target_version=4.2.1.6-106000012024042515, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-06-03T11:51:48.780+08:00, ob_data_disk_path=/data01/obdata}, listMap={stan dby_cluster_ids=[], 3.obuat.zone_names=[zone3, zone1, zone2], 3.obuat.zone1.host_ids=[7], cluster_ids=[4], primary_first_cluster_ids=[4], 3.obuat.zone3.server_ids=[10], 3.obuat.zone1.server_ids=[8], zone_names=[zone3, zone1, zone2], 3.obuat.zone2.server_ids=[9], 3.obuat.zone3.host_ids=[5], server_ids=[9], standby_first_cluster_ids=[4], staged_versions=[4.2.1.6-106000012024042515], 3.obuat.zone2.host_ids=[6], host_ids=[6], direct_upgrade_versions=[]}}, executor=172.16.185.114 2024-06-03 11:51:49.026 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : need to do nothing when rollback 2024-06-03 11:51:49.030 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py on primary cluster, version=4.2.1.6-106000012024042515 2024-06-03 11:51:49.033 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py 2024-06-03 11:51:49.058 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:51:49.072 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obuat, tenantName=sys, dbUser=root 2024-06-03 11:51:49.076 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:51:49.079 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:51:49.081 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 11:51:49.084 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2024-06-03 11:51:49.110 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:51:49.113 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 11:51:49.115 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2024-06-03 11:51:49.130 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=null, obClusterId=null, tenantName=sys, username=root, address=172.16.103.165, port=2881, database=oceanbase) 2024-06-03 11:51:49.133 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-06-03 11:51:52.135 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] init ob connector timeout, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), initTimeoutMillis=3000 2024-06-03 11:51:52.301 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.ocp.core.util.ExceptionUtils : Checked Exception: com.oceanbase.ocp.core.exception.UnexpectedException occurred with code error.ob.cluster.server.connect.failed, and args [obuat, 3, 172.16.103.165] 2024-06-03 11:51:52.305 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,5af8e5bdd18d] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Failed to connect to Cluster obuat:3 via observer 172.16.103.165 com.oceanbase.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_SERVER_CONNECT_FAILED, args=obuat,3,172.16.103.165 at sun.reflect.GeneratedConstructorAccessor1489.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at com.oceanbase.ocp.core.util.ExceptionUtils.newException(ExceptionUtils.java:96) at com.oceanbase.ocp.core.util.ExceptionUtils.throwException(ExceptionUtils.java:90) at com.oceanbase.ocp.core.util.ExceptionUtils.unExpected(ExceptionUtils.java:77) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:99) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:80) at com.oceanbase.ocp.service.task.business.cluster.ExecUpgradePostScriptTask.run(ExecUpgradePostScriptTask.java:66) at com.oceanbase.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.retry(JavaSubtaskRunner.java:76) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:35) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExec utor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 19837485, operation:RETRY, state: FAILED 2024-06-03 11:52:14.675 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=19837485, context=Context{parallelIdx=0, stringMap={ob_log_disk_path=/data01/obredo, task_instance_id=13682619, task_operation=retry, ob_run_path=/home/admin/oceanbase, zone_name=zone2, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, cluster_id=4, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade post script, watched_scenario=UPGRADE, sub_task_instance_id=19837485, primary_cluster_id=4, cluster_version=4.2.1.0, obuat.3.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"interface_center\":\"True\",\"luban_global_id_aduit\":\"True\",\"cloud_object_storage\":\"True\",\"universe_search\":\"True\",\"business_audit_platform\":\"True\",\"vchat_obdb\":\"True\",\"serialcode_center_pt\":\"True\",\"sys\":\"True\",\"mq_gateway\":\"True\",\"sales\":\"True\",\"trade_push\":\"True\",\"uuc_login_aduit\":\"True\"}","enable_rereplication":"True"}, cluster_name=obuat, mix_rpm_names={"zone3":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=3, target_version=4.2.1.6-106000012024042515, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-06-03T11:52:14.512+08:00, ob_data_disk_path=/data01/obdata}, listMap={stan dby_cluster_ids=[], 3.obuat.zone_names=[zone3, zone1, zone2], 3.obuat.zone1.host_ids=[7], cluster_ids=[4], primary_first_cluster_ids=[4], 3.obuat.zone3.server_ids=[10], 3.obuat.zone1.server_ids=[8], zone_names=[zone3, zone1, zone2], 3.obuat.zone2.server_ids=[9], 3.obuat.zone3.host_ids=[5], server_ids=[9], standby_first_cluster_ids=[4], staged_versions=[4.2.1.6-106000012024042515], 3.obuat.zone2.host_ids=[6], host_ids=[6], direct_upgrade_versions=[]}}, executor=172.16.185.114 2024-06-03 11:52:14.680 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : need to do nothing when rollback 2024-06-03 11:52:14.682 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py on primary cluster, version=4.2.1.6-106000012024042515 2024-06-03 11:52:14.684 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py 2024-06-03 11:52:14.710 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:52:14.722 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obuat, tenantName=sys, dbUser=root 2024-06-03 11:52:14.726 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:52:14.729 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:52:14.731 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 11:52:14.734 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2024-06-03 11:52:14.750 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:52:14.752 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 11:52:14.755 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2024-06-03 11:52:14.765 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=null, obClusterId=null, tenantName=sys, username=root, address=172.16.103.165, port=2881, database=oceanbase) 2024-06-03 11:52:14.768 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-06-03 11:52:17.770 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] init ob connector timeout, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), initTimeoutMillis=3000 2024-06-03 11:52:17.774 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.ocp.core.util.ExceptionUtils : Checked Exception: com.oceanbase.ocp.core.exception.UnexpectedException occurred with code error.ob.cluster.server.connect.failed, and args [obuat, 3, 172.16.103.165] 2024-06-03 11:52:17.777 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c120b48c3118] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Failed to connect to Cluster obuat:3 via observer 172.16.103.165 com.oceanbase.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_SERVER_CONNECT_FAILED, args=obuat,3,172.16.103.165 at sun.reflect.GeneratedConstructorAccessor1489.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at com.oceanbase.ocp.core.util.ExceptionUtils.newException(ExceptionUtils.java:96) at com.oceanbase.ocp.core.util.ExceptionUtils.throwException(ExceptionUtils.java:90) at com.oceanbase.ocp.core.util.ExceptionUtils.unExpected(ExceptionUtils.java:77) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:99) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:80) at com.oceanbase.ocp.service.task.business.cluster.ExecUpgradePostScriptTask.run(ExecUpgradePostScriptTask.java:66) at com.oceanbase.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.retry(JavaSubtaskRunner.java:76) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:35) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExec utor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 19837485, operation:RETRY, state: FAILED 2024-06-03 11:52:40.053 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=19837485, context=Context{parallelIdx=0, stringMap={ob_log_disk_path=/data01/obredo, task_instance_id=13682619, task_operation=retry, ob_run_path=/home/admin/oceanbase, zone_name=zone2, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, cluster_id=4, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade post script, watched_scenario=UPGRADE, sub_task_instance_id=19837485, primary_cluster_id=4, cluster_version=4.2.1.0, obuat.3.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"interface_center\":\"True\",\"luban_global_id_aduit\":\"True\",\"cloud_object_storage\":\"True\",\"universe_search\":\"True\",\"business_audit_platform\":\"True\",\"vchat_obdb\":\"True\",\"serialcode_center_pt\":\"True\",\"sys\":\"True\",\"mq_gateway\":\"True\",\"sales\":\"True\",\"trade_push\":\"True\",\"uuc_login_aduit\":\"True\"}","enable_rereplication":"True"}, cluster_name=obuat, mix_rpm_names={"zone3":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=3, target_version=4.2.1.6-106000012024042515, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-06-03T11:52:39.890+08:00, ob_data_disk_path=/data01/obdata}, listMap={stan dby_cluster_ids=[], 3.obuat.zone_names=[zone3, zone1, zone2], 3.obuat.zone1.host_ids=[7], cluster_ids=[4], primary_first_cluster_ids=[4], 3.obuat.zone3.server_ids=[10], 3.obuat.zone1.server_ids=[8], zone_names=[zone3, zone1, zone2], 3.obuat.zone2.server_ids=[9], 3.obuat.zone3.host_ids=[5], server_ids=[9], standby_first_cluster_ids=[4], staged_versions=[4.2.1.6-106000012024042515], 3.obuat.zone2.host_ids=[6], host_ids=[6], direct_upgrade_versions=[]}}, executor=172.16.185.114 2024-06-03 11:52:40.058 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : need to do nothing when rollback 2024-06-03 11:52:40.061 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py on primary cluster, version=4.2.1.6-106000012024042515 2024-06-03 11:52:40.064 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py 2024-06-03 11:52:40.082 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:52:40.097 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obuat, tenantName=sys, dbUser=root 2024-06-03 11:52:40.101 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:52:40.105 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:52:40.107 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 11:52:40.109 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2024-06-03 11:52:40.125 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 11:52:40.128 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 11:52:40.131 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2024-06-03 11:52:40.141 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=null, obClusterId=null, tenantName=sys, username=root, address=172.16.103.165, port=2881, database=oceanbase) 2024-06-03 11:52:40.143 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-06-03 11:52:43.145 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] init ob connector timeout, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), initTimeoutMillis=3000 2024-06-03 11:52:43.147 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.ocp.core.util.ExceptionUtils : Checked Exception: com.oceanbase.ocp.core.exception.UnexpectedException occurred with code error.ob.cluster.server.connect.failed, and args [obuat, 3, 172.16.103.165] 2024-06-03 11:52:43.149 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,9ab6999c793e] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Failed to connect to Cluster obuat:3 via observer 172.16.103.165 com.oceanbase.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_SERVER_CONNECT_FAILED, args=obuat,3,172.16.103.165 at sun.reflect.GeneratedConstructorAccessor1489.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at com.oceanbase.ocp.core.util.ExceptionUtils.newException(ExceptionUtils.java:96) at com.oceanbase.ocp.core.util.ExceptionUtils.throwException(ExceptionUtils.java:90) at com.oceanbase.ocp.core.util.ExceptionUtils.unExpected(ExceptionUtils.java:77) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:99) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:80) at com.oceanbase.ocp.service.task.business.cluster.ExecUpgradePostScriptTask.run(ExecUpgradePostScriptTask.java:66) at com.oceanbase.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.retry(JavaSubtaskRunner.java:76) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:35) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExec utor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 19837485, operation:RETRY, state: FAILED 2024-06-03 12:11:38.500 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=19837485, context=Context{parallelIdx=0, stringMap={ob_log_disk_path=/data01/obredo, task_instance_id=13682619, task_operation=retry, ob_run_path=/home/admin/oceanbase, zone_name=zone2, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, cluster_id=4, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade post script, watched_scenario=UPGRADE, sub_task_instance_id=19837485, primary_cluster_id=4, cluster_version=4.2.1.0, obuat.3.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"interface_center\":\"True\",\"luban_global_id_aduit\":\"True\",\"cloud_object_storage\":\"True\",\"universe_search\":\"True\",\"business_audit_platform\":\"True\",\"vchat_obdb\":\"True\",\"serialcode_center_pt\":\"True\",\"sys\":\"True\",\"mq_gateway\":\"True\",\"sales\":\"True\",\"trade_push\":\"True\",\"uuc_login_aduit\":\"True\"}","enable_rereplication":"True"}, cluster_name=obuat, mix_rpm_names={"zone3":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=3, target_version=4.2.1.6-106000012024042515, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-06-03T12:11:38.304+08:00, ob_data_disk_path=/data01/obdata}, listMap={stan dby_cluster_ids=[], 3.obuat.zone_names=[zone3, zone1, zone2], 3.obuat.zone1.host_ids=[7], cluster_ids=[4], primary_first_cluster_ids=[4], 3.obuat.zone3.server_ids=[10], 3.obuat.zone1.server_ids=[8], zone_names=[zone3, zone1, zone2], 3.obuat.zone2.server_ids=[9], 3.obuat.zone3.host_ids=[5], server_ids=[9], standby_first_cluster_ids=[4], staged_versions=[4.2.1.6-106000012024042515], 3.obuat.zone2.host_ids=[6], host_ids=[6], direct_upgrade_versions=[]}}, executor=172.16.185.114 2024-06-03 12:11:38.505 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : need to do nothing when rollback 2024-06-03 12:11:38.508 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py on primary cluster, version=4.2.1.6-106000012024042515 2024-06-03 12:11:38.511 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py 2024-06-03 12:11:38.527 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 12:11:38.540 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obuat, tenantName=sys, dbUser=root 2024-06-03 12:11:38.543 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 12:11:38.546 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 12:11:38.548 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 12:11:38.551 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2024-06-03 12:11:38.570 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 12:11:38.572 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 12:11:38.576 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2024-06-03 12:11:38.591 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=null, obClusterId=null, tenantName=sys, username=root, address=172.16.103.165, port=2881, database=oceanbase) 2024-06-03 12:11:38.594 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-06-03 12:11:41.596 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] init ob connector timeout, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), initTimeoutMillis=3000 2024-06-03 12:11:41.600 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.ocp.core.util.ExceptionUtils : Checked Exception: com.oceanbase.ocp.core.exception.UnexpectedException occurred with code error.ob.cluster.server.connect.failed, and args [obuat, 3, 172.16.103.165] 2024-06-03 12:11:41.603 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,790bd25dd9ab] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Failed to connect to Cluster obuat:3 via observer 172.16.103.165 com.oceanbase.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_SERVER_CONNECT_FAILED, args=obuat,3,172.16.103.165 at sun.reflect.GeneratedConstructorAccessor1489.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at com.oceanbase.ocp.core.util.ExceptionUtils.newException(ExceptionUtils.java:96) at com.oceanbase.ocp.core.util.ExceptionUtils.throwException(ExceptionUtils.java:90) at com.oceanbase.ocp.core.util.ExceptionUtils.unExpected(ExceptionUtils.java:77) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:99) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:80) at com.oceanbase.ocp.service.task.business.cluster.ExecUpgradePostScriptTask.run(ExecUpgradePostScriptTask.java:66) at com.oceanbase.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.retry(JavaSubtaskRunner.java:76) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:35) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExec utor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 19837485, operation:RETRY, state: FAILED 2024-06-03 12:15:57.207 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=19837485, context=Context{parallelIdx=0, stringMap={ob_log_disk_path=/data01/obredo, task_instance_id=13682619, task_operation=retry, ob_run_path=/home/admin/oceanbase, zone_name=zone2, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, cluster_id=4, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade post script, watched_scenario=UPGRADE, sub_task_instance_id=19837485, primary_cluster_id=4, cluster_version=4.2.1.0, obuat.3.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"interface_center\":\"True\",\"luban_global_id_aduit\":\"True\",\"cloud_object_storage\":\"True\",\"universe_search\":\"True\",\"business_audit_platform\":\"True\",\"vchat_obdb\":\"True\",\"serialcode_center_pt\":\"True\",\"sys\":\"True\",\"mq_gateway\":\"True\",\"sales\":\"True\",\"trade_push\":\"True\",\"uuc_login_aduit\":\"True\"}","enable_rereplication":"True"}, cluster_name=obuat, mix_rpm_names={"zone3":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=3, target_version=4.2.1.6-106000012024042515, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-06-03T12:15:57.051+08:00, ob_data_disk_path=/data01/obdata}, listMap={stan dby_cluster_ids=[], 3.obuat.zone_names=[zone3, zone1, zone2], 3.obuat.zone1.host_ids=[7], cluster_ids=[4], primary_first_cluster_ids=[4], 3.obuat.zone3.server_ids=[10], 3.obuat.zone1.server_ids=[8], zone_names=[zone3, zone1, zone2], 3.obuat.zone2.server_ids=[9], 3.obuat.zone3.host_ids=[5], server_ids=[9], standby_first_cluster_ids=[4], staged_versions=[4.2.1.6-106000012024042515], 3.obuat.zone2.host_ids=[6], host_ids=[6], direct_upgrade_versions=[]}}, executor=172.16.185.114 2024-06-03 12:15:57.212 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : need to do nothing when rollback 2024-06-03 12:15:57.215 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py on primary cluster, version=4.2.1.6-106000012024042515 2024-06-03 12:15:57.216 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py 2024-06-03 12:15:57.232 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 12:15:57.244 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obuat, tenantName=sys, dbUser=root 2024-06-03 12:15:57.247 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 12:15:57.251 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 12:15:57.254 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 12:15:57.256 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2024-06-03 12:15:57.271 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 12:15:57.275 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 12:15:57.278 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2024-06-03 12:15:57.287 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=null, obClusterId=null, tenantName=sys, username=root, address=172.16.103.165, port=2881, database=oceanbase) 2024-06-03 12:15:57.290 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-06-03 12:16:00.292 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] init ob connector timeout, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), initTimeoutMillis=3000 2024-06-03 12:16:00.295 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.ocp.core.util.ExceptionUtils : Checked Exception: com.oceanbase.ocp.core.exception.UnexpectedException occurred with code error.ob.cluster.server.connect.failed, and args [obuat, 3, 172.16.103.165] 2024-06-03 12:16:00.298 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,c497d00ceb1f] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Failed to connect to Cluster obuat:3 via observer 172.16.103.165 com.oceanbase.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_SERVER_CONNECT_FAILED, args=obuat,3,172.16.103.165 at sun.reflect.GeneratedConstructorAccessor1489.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at com.oceanbase.ocp.core.util.ExceptionUtils.newException(ExceptionUtils.java:96) at com.oceanbase.ocp.core.util.ExceptionUtils.throwException(ExceptionUtils.java:90) at com.oceanbase.ocp.core.util.ExceptionUtils.unExpected(ExceptionUtils.java:77) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:99) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:80) at com.oceanbase.ocp.service.task.business.cluster.ExecUpgradePostScriptTask.run(ExecUpgradePostScriptTask.java:66) at com.oceanbase.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.retry(JavaSubtaskRunner.java:76) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:35) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExec utor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 19837485, operation:RETRY, state: FAILED 2024-06-03 16:23:27.735 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=19837485, context=Context{parallelIdx=0, stringMap={ob_log_disk_path=/data01/obredo, task_instance_id=13682619, task_operation=retry, ob_run_path=/home/admin/oceanbase, zone_name=zone2, ob_install_path=/home/admin/oceanbase, ob_sql_port=2881, cluster_id=4, ocpagent_service_name=agent, ob_disk_path_style=DEFAULT, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Execute upgrade post script, watched_scenario=UPGRADE, sub_task_instance_id=19837485, primary_cluster_id=4, cluster_version=4.2.1.0, obuat.3.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"interface_center\":\"True\",\"luban_global_id_aduit\":\"True\",\"cloud_object_storage\":\"True\",\"universe_search\":\"True\",\"business_audit_platform\":\"True\",\"vchat_obdb\":\"True\",\"serialcode_center_pt\":\"True\",\"sys\":\"True\",\"mq_gateway\":\"True\",\"sales\":\"True\",\"trade_push\":\"True\",\"uuc_login_aduit\":\"True\"}","enable_rereplication":"True"}, cluster_name=obuat, mix_rpm_names={"zone3":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone2":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm","zone1":"oceanbase-ce-4.2.1.6-106000012024042515.el7.x86_64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=3, target_version=4.2.1.6-106000012024042515, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.2.1.6-106000012024042515.el7.x86_64.rpm","oceanbase-ce-utils-4.2.1.6-106000012024042515.el7.x86_64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-06-03T16:23:27.560+08:00, ob_data_disk_path=/data01/obdata}, listMap={stan dby_cluster_ids=[], 3.obuat.zone_names=[zone3, zone1, zone2], 3.obuat.zone1.host_ids=[7], cluster_ids=[4], primary_first_cluster_ids=[4], 3.obuat.zone3.server_ids=[10], 3.obuat.zone1.server_ids=[8], zone_names=[zone3, zone1, zone2], 3.obuat.zone2.server_ids=[9], 3.obuat.zone3.host_ids=[5], server_ids=[9], standby_first_cluster_ids=[4], staged_versions=[4.2.1.6-106000012024042515], 3.obuat.zone2.host_ids=[6], host_ids=[6], direct_upgrade_versions=[]}}, executor=172.16.185.114 2024-06-03 16:23:27.741 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : need to do nothing when rollback 2024-06-03 16:23:27.744 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py on primary cluster, version=4.2.1.6-106000012024042515 2024-06-03 16:23:27.749 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.o.s.t.b.c.ExecUpgradePostScriptTask : execute upgrade_post.py 2024-06-03 16:23:27.769 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 16:23:27.783 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=obuat, tenantName=sys, dbUser=root 2024-06-03 16:23:27.790 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 16:23:27.795 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 16:23:27.800 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 16:23:27.805 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2024-06-03 16:23:27.824 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:172.16.103.202, sql port:2881 2024-06-03 16:23:27.827 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-06-03 16:23:27.831 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2024-06-03 16:23:27.849 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=null, obClusterId=null, tenantName=sys, username=root, address=172.16.103.165, port=2881, database=oceanbase) 2024-06-03 16:23:27.859 INFO 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-06-03 16:23:30.863 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] init ob connector timeout, connectProperties=ConnectProperties(connectionMode=direct, address=172.16.103.165, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), initTimeoutMillis=3000 2024-06-03 16:23:30.866 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.ocp.core.util.ExceptionUtils : Checked Exception: com.oceanbase.ocp.core.exception.UnexpectedException occurred with code error.ob.cluster.server.connect.failed, and args [obuat, 3, 172.16.103.165] 2024-06-03 16:23:30.869 ERROR 54383 --- [pool-manual-subtask-executor14,2aa5e462d52e4eca,63510d664901] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Failed to connect to Cluster obuat:3 via observer 172.16.103.165 com.oceanbase.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_SERVER_CONNECT_FAILED, args=obuat,3,172.16.103.165 at sun.reflect.GeneratedConstructorAccessor1489.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at com.oceanbase.ocp.core.util.ExceptionUtils.newException(ExceptionUtils.java:96) at com.oceanbase.ocp.core.util.ExceptionUtils.throwException(ExceptionUtils.java:90) at com.oceanbase.ocp.core.util.ExceptionUtils.unExpected(ExceptionUtils.java:77) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:99) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:80) at com.oceanbase.ocp.service.task.business.cluster.ExecUpgradePostScriptTask.run(ExecUpgradePostScriptTask.java:66) at com.oceanbase.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.retry(JavaSubtaskRunner.java:76) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:35) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExec utor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 19837485, operation:RETRY, state: FAILED