2025-02-27 10:11:13.676 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.c.t.e.runner.JavaSubtaskRunner : Run subtask, id=11266481, context=Context{parallelIdx=0, stringMap={cluster_version=4.1.0.1, cluster_name=uatbg_ob01, subtask_splitter=primary_first_cluster_ids, uatbg_ob01.6.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"True","enable_rereplication":"True"}, task_instance_id=10027895, task_operation=execute, target_version=4.2.1.2-102000042023120514, cluster_id=2000003, ocpagent_service_name=agent, operating_system=el7, target_cluster_status=RUNNING, latest_execution_start_time=2025-02-27T10:11:13.581+08:00, sub_task_instance_name=Execute upgrade checker script, watched_scenario=UPGRADE, sub_task_instance_id=11266481, primary_cluster_id=2000003}, listMap={standby_cluster_ids=[], standby_first_cluster_ids=[2000003], 6.uatbg_ob01.zone_names=[zone1], staged_versions=[4.2.1.2-102000042023120514], cluster_ids=[2000003], primary_first_cluster_ids=[2000003], 6.uatbg_ob01.host_ids=[2000003], direct_upgrade_versions=[], 6.uatbg_ob01.server_ids=[2000003]}}, executor=10.106.127.16 2025-02-27 10:11:13.679 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=uatbg_ob01 2025-02-27 10:11:13.681 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=uatbg_ob01, obClusterId=6, tenantName=sys, username=ocp_monitor, address=10.106.21.1, port=2881, database=oceanbase) 2025-02-27 10:11:13.684 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=uatbg_ob01, obClusterId=6, tenantName=sys, username=ocp_monitor, address=10.106.21.1, port=2881, database=oceanbase) 2025-02-27 10:11:13.688 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] com.alibaba.druid.pool.DruidDataSource : {dataSource-31860} closing ... 2025-02-27 10:11:13.720 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] com.alibaba.druid.pool.DruidDataSource : {dataSource-31860} closed 2025-02-27 10:11:13.727 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=uatbg_ob01, obClusterId=6, tenantName=uatcxob01, username=root, address=10.106.21.1, port=2881, database=oceanbase) 2025-02-27 10:11:13.737 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=uatbg_ob01, obClusterId=6, tenantName=uatcxob01, username=root, address=10.106.21.1, port=2881, database=oceanbase) 2025-02-27 10:11:13.742 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] com.alibaba.druid.pool.DruidDataSource : {dataSource-31861} closing ... 2025-02-27 10:11:13.753 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] com.alibaba.druid.pool.DruidDataSource : {dataSource-31861} closed 2025-02-27 10:11:13.758 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=uatbg_ob01, obClusterId=6, tenantName=sys, username=root, address=10.106.21.1, port=2881, database=oceanbase) 2025-02-27 10:11:13.767 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=uatbg_ob01, obClusterId=6, tenantName=sys, username=root, address=10.106.21.1, port=2881, database=oceanbase) 2025-02-27 10:11:13.775 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] com.alibaba.druid.pool.DruidDataSource : {dataSource-31858} closing ... 2025-02-27 10:11:13.798 ERROR 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.a.d.stat.DruidDataSourceStatManager : unregister mbean error javax.management.InstanceNotFoundException: com.alibaba.druid:type=DruidDataSource,id=ob-connector-datasource_direct_sys_root_10.106.21.1_2881 at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1095) at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:427) at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415) at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546) at com.alibaba.druid.stat.DruidDataSourceStatManager.removeDataSource(DruidDataSourceStatManager.java:202) at com.alibaba.druid.pool.DruidDataSource$2.run(DruidDataSource.java:2172) at java.security.AccessController.doPrivileged(Native Method) at com.alibaba.druid.pool.DruidDataSource.unregisterMbean(DruidDataSource.java:2168) at com.alibaba.druid.pool.DruidDataSource.close(DruidDataSource.java:2124) at com.alibaba.druid.util.JdbcUtils.close(JdbcUtils.java:131) at com.oceanbase.ocp.obsdk.connector.impl.AbstractConnector.close(AbstractConnector.java:110) at com.oceanbase.ocp.obsdk.connector.impl.DefaultConnector.close(DefaultConnector.java:14) at com.oceanbase.ocp.obsdk.connector.ObConnectorHolder$1.delete(ObConnectorHolder.java:70) at com.oceanbase.ocp.obsdk.connector.ObConnectorHolder$1.delete(ObConnectorHolder.java:45) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$removeWithWriter$9(BoundedLocalCache.java:2147) at java.util.concurrent.ConcurrentHashMap.computeIfPresent(ConcurrentHashMap.java:1769) at com.github.benmanes.caffeine.cache.BoundedLocalCache.removeWithWriter(BoundedLocalCache.java:2137) at com.github.benmanes.caffeine.cache.BoundedLocalCache.remove(BoundedLocalCache.java:2080) at com.github.benmanes.caffeine.cache.LocalManualCache.invalidate (LocalManualCache.java:140) at com.oceanbase.ocp.obsdk.connector.ObConnectorHolder.lambda$invalidate$2(ObConnectorHolder.java:154) at java.lang.Iterable.forEach(Iterable.java:75) at com.oceanbase.ocp.obsdk.connector.ObConnectorHolder.invalidate(ObConnectorHolder.java:151) at com.oceanbase.ocp.obsdk.connector.ObConnectorHolder.invalidate(ObConnectorHolder.java:137) at com.oceanbase.ocp.obsdk.connector.ObConnectors.invalidate(ObConnectors.java:122) at com.oceanbase.ocp.service.task.business.cluster.ExecUpgradeCheckerScriptTask.run(ExecUpgradeCheckerScriptTask.java:52) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.execute(JavaSubtaskRunner.java:64) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:32) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) 2025-02-27 10:11:13.804 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] com.alibaba.druid.pool.DruidDataSource : {dataSource-31858} closed 2025-02-27 10:11:13.807 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] o.o.s.t.b.c.ExecUpgradeCheckerScriptTask : execute upgrade_checker.py, clusterId=2000003, version=4.2.1.2-102000042023120514 2025-02-27 10:11:13.833 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=uatbg_ob01, obClusterId=6, tenantName=sys, username=root, address=10.106.21.1, port=2881, database=oceanbase) 2025-02-27 10:11:13.835 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=null, port=null, obsAddrList=[ObServerAddr(address=10.106.21.1, port=2881)], username=root, tenantName=sys, clusterName=uatbg_ob01, obClusterId=6, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2025-02-27 10:11:13.846 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] create new ob connector success 2025-02-27 10:11:13.850 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.106.21.1, sql port:2881 2025-02-27 10:11:13.860 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=uatbg_ob01, tenantName=sys, dbUser=root 2025-02-27 10:11:13.863 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.106.21.1, sql port:2881 2025-02-27 10:11:13.866 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.106.21.1, sql port:2881 2025-02-27 10:11:13.868 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-02-27 10:11:13.871 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `svr_ip`, `svr_port`, `zone`, `role` FROM `CDB_OB_LS_LOCATIONS` WHERE `tenant_id` = 1 AND `ls_id` = 1 2025-02-27 10:11:13.893 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.106.21.1, sql port:2881 2025-02-27 10:11:13.896 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-02-27 10:11:13.900 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select version() 2025-02-27 10:11:13.925 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=null, obClusterId=null, tenantName=sys, username=root, address=10.106.21.1, port=2881, database=oceanbase) 2025-02-27 10:11:13.928 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=10.106.21.1, port=2881, obsAddrList=null, username=root, tenantName=sys, clusterName=null, obClusterId=null, proxy=null, odpSocketConfig=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2025-02-27 10:11:13.938 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ObConnectors : [obsdk] create new ob connector success 2025-02-27 10:11:13.942 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.106.21.1, sql port:2881 2025-02-27 10:11:13.944 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.106.21.1, sql port:2881 2025-02-27 10:11:13.946 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-02-27 10:11:13.949 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2025-02-27 10:11:13.955 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : execute script upgrade_checker.py on cluster, clusterId=2000003, obClusterId=6, zone=null, clusterType=PRIMARY, rpmName=oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm 2025-02-27 10:11:13.960 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=2000003 2025-02-27 10:11:13.965 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 2000003 2025-02-27 10:11:13.981 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.106.21.1:62888/api/v1/file/exists, request body:GetFileExistsRequest(filePath=rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py), params:null 2025-02-27 10:11:13.990 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.s.t.b.c.helper.UpgradeTaskHelper : [runScript]: generate agent async task token:dadedb0a-e28b-4988-9eb9-88623dd2781d 2025-02-27 10:11:13.995 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=2000003 2025-02-27 10:11:13.999 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 2000003 2025-02-27 10:11:14.015 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.106.21.1:62888/api/v1/ob/upgrade/runScript, request body:RunUpgradeScriptRequest(scriptPath=rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py, arguments=RunUpgradeScriptArgument(host=127.0.0.1, port=2881, logFile=4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log, zone=null, cmdTimeout=0)), params:null 2025-02-27 10:11:14.262 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.106.21.1:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dadedb0a-e28b-4988-9eb9-88623dd2781d), params:null 2025-02-27 10:11:14.265 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,, 2025-02-27 10:11:14.269 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.common.lang.pattern.Retry : wait for 15 seconds 2025-02-27 10:11:29.272 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.106.21.1:62888/api/v1/task/status, request body:GetTaskStatusRequest(taskToken=dadedb0a-e28b-4988-9eb9-88623dd2781d), params:null 2025-02-27 10:11:29.279 INFO 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.s.task.util.AgentAsyncTaskHelper : try to request task result(EXECUTE), result:false,null,OcpAgentError: code = 1500, message = Execute shell command failed: failed to execute command: Command{user=root, program=sh, outputType=combined, cmd=export PYTHONPATH=/home/admin/ocp_agent/site-packages:$PYTHONPATH && python /tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log, timeout=30m0s}, exitCode: 1, output: [2025-02-27 10:11:14] INFO upgrade_checker.py:675 parameters from cmd: host="127.0.0.1", port=2881, user="root@sys", password=xxx timeout="0", log-file="/tmp/4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log" [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:325 check observer version success, version = 4.1.0.1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from oceanbase.__all_virtual_tenant_parameter_info where name='compatible', rowcount = 2 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as unsync_cnt from GV$OB_LOG_STAT where in_sync = 'NO', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:385 check paxos replica success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as cnt from DBA_OB_TENANT_JOBS where job_status='INPROGRESS' and result_code is null, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: sele ct count(1) as rebalance_task_cnt from CDB_OB_LS_REPLICA_TASKS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:397 check rebalance task success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE'), rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:409 check cluster tablet major status success, cluster_version=[17179934721] [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) as count from DBA_OB_TENANTS where status != 'NORMAL', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:423 check tenant status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) as count from oceanbase.__all_virtual_tenant_info where tenant_role != 'PRIMARY' and tenant_role != 'STANDBY', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:433 check tenant info success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_RESTORE_PROGRESS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:442 check restore job success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from __all_virtual_ddl_task_status, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:488 check ddl task execut status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_BACKUP_JOBS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:499 check backup job suc cess [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status != "active"), rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:578 check observer status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:585 check schema status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct(substring_index(build_version, '_', 1)) from __all_server, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:571 check server version success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select tenant_name from oceanbase.DBA_OB_TENANTS, rowcount = 3 [2025-02-27 10:11:14] INFO upgrade_checker.py:595 check special tenant name success [2025-02-27 10:11:14] ERROR upgrade_checker.py:644 run error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 641, in do_check check_fail_list() File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/ admin/oceanbase/etc/upgrade_checker.py", line 601, in check_fail_list raise MyError(error_msg) MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' [2025-02-27 10:11:14] ERROR upgrade_checker.py:653 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 645, in do_check raise e MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' [2025-02-27 10:11:14] ERROR upgrade_checker.py:681 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 676, in do_check(host, port, user, password, timeout, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 654, in do_check raise e MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 682, in raise e __main__.MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' , error: exit status 1, 2025-02-27 10:11:29.312 ERROR 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.s.task.util.AgentAsyncTaskHelper : failed to execute async task, task token: dadedb0a-e28b-4988-9eb9-88623dd2781d, err: OcpAgentError: code = 1500, message = Execute shell command failed: failed to execute command: Command{user=root, program=sh, outputType=combined, cmd=export PYTHONPATH=/home/admin/ocp_agent/site-packages:$PYTHONPATH && python /tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log, timeout=30m0s}, exitCode: 1, output: [2025-02-27 10:11:14] INFO upgrade_checker.py:675 parameters from cmd: host="127.0.0.1", port=2881, user="root@sys", password=xxx timeout="0", log-file="/tmp/4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log" [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:325 check observer version success, version = 4.1.0.1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from oceanbase.__all_virtual_tenant_parameter_info where name='compatible', rowcount = 2 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as unsync_cnt from GV$OB_LOG_STAT where in_sync = 'NO', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:385 check paxos replica success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as cnt from DBA_OB_TENANT_JOBS where job_status='INPROGRESS' and result_code is null, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as rebalance_task_cnt from CDB_OB_LS_REPLICA_TASKS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:397 check rebalance task success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE'), rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:409 check cluster tablet major status success, cluster_version=[17179934721] [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) as count from DBA_OB_TENANTS where status != 'NORMAL', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:423 check tenant status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) as count from oceanbase.__all_virtual_tenant_info where tenant_role != 'PRIMARY' and tenant_role != 'STANDBY', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:433 check tenant info success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_RESTORE_PROGRESS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:442 check restore job success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from __all_virtual_ddl_task_status, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:488 check ddl task execut status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_BACKUP_JOBS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_check er.py:499 check backup job success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status != "active"), rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:578 check observer status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:585 check schema status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct(substring_index(build_version, '_', 1)) from __all_server, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:571 check server version success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select tenant_name from oceanbase.DBA_OB_TENANTS, rowcount = 3 [2025-02-27 10:11:14] INFO upgrade_checker.py:595 check special tenant name success [2025-02-27 10:11:14] ERROR upgrade_checker.py:644 run error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 641, in do_check check_fail_list() File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042 023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 601, in check_fail_list raise MyError(error_msg) MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' [2025-02-27 10:11:14] ERROR upgrade_checker.py:653 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 645, in do_check raise e MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' [2025-02-27 10:11:14] ERROR upgrade_checker.py:681 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 676, in do_check(host, port, user, password, timeout, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 654, in do_check raise e MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 682, in raise e __main__.MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' , error: exit status 1 2025-02-27 10:11:29.319 ERROR 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.ocp.core.util.ExceptionUtils : Checked Exception: com.oceanbase.ocp.core.exception.UnexpectedException occurred with code error.agent.task.failed, and args [dadedb0a-e28b-4988-9eb9-88623dd2781d, OcpAgentError: code = 1500, message = Execute shell command failed: failed to execute command: Command{user=root, program=sh, outputType=combined, cmd=export PYTHONPATH=/home/admin/ocp_agent/site-packages:$PYTHONPATH && python /tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log, timeout=30m0s}, exitCode: 1, output: [2025-02-27 10:11:14] INFO upgrade_checker.py:675 parameters from cmd: host="127.0.0.1", port=2881, user="root@sys", password=xxx timeout="0", log-file="/tmp/4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log" [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:325 check observer version success, version = 4.1.0.1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from oceanbase.__all_virtual_tenant_parameter_info where name='compatible', rowcount = 2 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as unsync_cnt from GV$OB_LOG_STAT where in_sync = 'NO', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:385 check paxos replica success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as cnt from DBA_OB_TENANT_JOBS where job_status='INPROGRESS' and resu lt_code is null, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as rebalance_task_cnt from CDB_OB_LS_REPLICA_TASKS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:397 check rebalance task success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE'), rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:409 check cluster tablet major status success, cluster_version=[17179934721] [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) as count from DBA_OB_TENANTS where status != 'NORMAL', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:423 check tenant status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) as count from oceanbase.__all_virtual_tenant_info where tenant_role != 'PRIMARY' and tenant_role != 'STANDBY', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:433 check tenant info success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_RESTORE_PROGRESS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:442 check restore job success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from __all_virtual_ddl_task_status, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:488 check ddl task execut status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_BACKUP_JOBS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:499 check backup job success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status != "active"), rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:578 check observer status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:585 check schema status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct(substring_index(build_version, '_', 1)) from __all_server, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:571 check server version success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select tenant_name from oceanbase.DBA_OB_TENANTS, rowcount = 3 [2025-02-27 10:11:14] INFO upgrade_checker.py:595 check special tenant name success [2025-02-27 10:11:14] ERROR upgrade_checker.py:644 run error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 641, in do_check check_fail_list() File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 601, in check_fail_list raise MyError(error_msg) MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' [2025-02-27 10:11:14] ERROR upgrade_checker.py:653 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 645, in do_check raise e MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' [2025-02-27 10:11:14] ERROR upgrade_checker.py:681 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 676, in do_check(host, port, user, password, timeout, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 654, in do_check raise e MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 682, in raise e __main__.MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' , error: exit status 1] 2025-02-27 10:11:29.326 ERROR 29947 --- [pool-manual-subtask-executor15,f08d46832ed34c69,5e418839d061] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : OCP-Agent task dadedb0a-e28b-4988-9eb9-88623dd2781d failed, error message: OcpAgentError: code = 1500, message = Execute shell command failed: failed to execute command: Command{user=root, program=sh, outputType=combined, cmd=export PYTHONPATH=/home/admin/ocp_agent/site-packages:$PYTHONPATH && python /tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log, timeout=30m0s}, exitCode: 1, output: [2025-02-27 10:11:14] INFO upgrade_checker.py:675 parameters from cmd: host="127.0.0.1", port=2881, user="root@sys", password=xxx timeout="0", log-file="/tmp/4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log" [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:325 check observer version success, version = 4.1.0.1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from oceanbase.__all_virtual_tenant_parameter_info where name='compatible', rowcount = 2 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as unsync_cnt from GV$OB_LOG_STAT where in_sync = 'NO', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:385 check paxos replica success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as cnt from DBA_OB_TENANT_JOBS where job_status='INPROGRESS' and result_code is null, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as rebalance_task_cnt from CDB_OB_LS_REPLICA_TASKS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:397 check rebalance task success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE'), rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:409 check cluster tablet major status success, cluster_version=[17179934721] [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) as count from DBA_OB_TENANTS where status != 'NORMAL', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:423 check tenant status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) as count from oceanbase.__all_virtual_tenant_info where tenant_role != 'PRIMARY' and tenant_role != 'STANDBY', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:433 check tenant info success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_RESTORE_PROGRESS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:442 check restore job success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from __all_virtual_ddl_task_status, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:488 check ddl task execut status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_BACKUP_JOBS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:499 check backup job success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status != "active"), rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:578 check observer status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:585 check schema status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct(substring_index(build_version, '_', 1)) from __all_server, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:571 check server version success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select tenant_name from oceanbase.DBA_OB_TENANTS, rowcount = 3 [2025-02-27 10:11:14] INFO upgrade_checker.py:595 check special tenant name success [2025-02-27 10:11:14] ERROR upgrade_checker.py:644 run error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 641, in do_check check_fail_list() File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514. el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 601, in check_fail_list raise MyError(error_msg) MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' [2025-02-27 10:11:14] ERROR upgrade_checker.py:653 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 645, in do_check raise e MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' [2025-02-27 10:11:14] ERROR upgrade_checker.py:681 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 676, in do_check(host, port, user, password, timeout, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 654, in do_check raise e MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 682, in raise e __main__.MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' , error: exit status 1 com.oceanbase.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=AGENT_TASK_FAILED, args=dadedb0a-e28b-4988-9eb9-88623dd2781d,OcpAgentError: code = 1500, message = Execute shell command failed: failed to execute command: Command{user=root, program=sh, outputType=combined, cmd=export PYTHONPATH=/home/admin/ocp_agent/site-packages:$PYTHONPATH && python /tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py -h127.0.0.1 -P2881 -uroot@sys --password=xxx -l /tmp/4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log, timeout=30m0s}, exitCode: 1, output: [2025-02-27 10:11:14] INFO upgrade_checker.py:675 parameters from cmd: host="127.0.0.1", port=2881, user="root@sys", password=xxx timeout="0", log-file="/tmp/4.2.1.2-102000042023120514_upgrade_checker_20250227101113.log" [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:325 check observer version success, version = 4.1.0.1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from oceanbase.__all_virtual_tenant_parameter_info where name='compatible', rowcount = 2 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as unsync_cnt from GV$OB_LOG_STAT where in_sync = 'NO', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:385 check paxos replica success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as cnt from DBA_OB_TENANT_JOBS where job_status='INPROGRESS' and result_code is null, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) as rebalance_task_cnt from CDB_OB_LS_REPLICA_TASKS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:397 check rebalance task success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE'), rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:409 check cluster tablet major status success, cluster_version=[17179934721] [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) as count from DBA_OB_TENANTS where status != 'NORMAL', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:423 check tenant status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) as count from oceanbase.__all_virtual_tenant_info where tenant_role != 'PRIMARY' and tenant_role != 'STANDBY', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:433 check tenant info success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_RESTORE_PROGRESS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:442 check restore job success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from __all_virtual_ddl_task_status, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:488 check ddl task execut status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(1) from CDB_OB_BACKUP_JOBS, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:499 check backup job success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct value from GV$OB_PARAMETERS where name='min_observer_version', rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status != "act ive"), rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:578 check observer status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:585 check schema status success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select distinct(substring_index(build_version, '_', 1)) from __all_server, rowcount = 1 [2025-02-27 10:11:14] INFO upgrade_checker.py:571 check server version success [2025-02-27 10:11:14] INFO upgrade_checker.py:45 succeed to execute query: select tenant_name from oceanbase.DBA_OB_TENANTS, rowcount = 3 [2025-02-27 10:11:14] INFO upgrade_checker.py:595 check special tenant name success [2025-02-27 10:11:14] ERROR upgrade_checker.py:644 run error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 641, in do_check check_fail_list() File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 601, in check_fail_list raise MyError(error_msg) MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' [2025-02-27 10:11:14] ERROR upgrade_checker.py:653 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 645, in do_check raise e MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' [2025-02-27 10:11:14] ERROR upgrade_checker.py:681 normal error Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514. el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 676, in do_check(host, port, user, password, timeout, upgrade_params) File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 654, in do_check raise e MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' Traceback (most recent call last): File "/tmp/rpms/extract/oceanbase-ce-4.2.1.2-102000042023120514.el7.x86_64.rpm/home/admin/oceanbase/etc/upgrade_checker.py", line 682, in raise e __main__.MyError: 'upgrade checker failed with 1 reasons: [compatible is not sync] ' , error: exit status 1 at com.oceanbase.ocp.core.i18n.ErrorCodes$Kind$10.exception(ErrorCodes.java:1570) at com.oceanbase.ocp.core.i18n.ErrorCodes.exception(ErrorCodes.java:1425) at com.oceanbase.ocp.core.util.ExceptionUtils.newException(ExceptionUtils.java:169) at com.oceanbase.ocp.core.util.ExceptionUtils.throwException(ExceptionUtils.java:162) at com.oceanbase.ocp.service.task.util.AgentAsyncTaskHelper.checkSuccess(AgentAsyncTaskHelper.java:229) at com.oceanbase.ocp.service.task.util.AgentAsyncTaskHelper.waitForExecuteFinish(AgentAsyncTaskHelper.java:175) at com.oceanbase.ocp.service.task.util.AgentAsyncTaskHelper.runUpgradeScript(AgentAsyncTaskHelper.java:130) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:160) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:105) at com.oceanbase.ocp.service.task.business.cluster.helper.UpgradeTaskHelper.runScript(UpgradeTaskHelper.java:80) at com.oceanbase.ocp.service.task.business.cluster.ExecUpgradeCheckerScriptTask.run(ExecUpgradeCheckerScriptTask.java:60) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.execute(JavaSubtaskRunner.java:64) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:32) at com.oceanbase.ocp.core.task.engine .runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 11266481, operation:EXECUTE, state: FAILED