Set state for subtask: 3001599, operation:EXECUTE, state: FAILED 2024-08-09 12:27:37.720 WARN 699463 --- [pool-subtask-executor30,4f3345b78e6f435e,c0219a3c648d] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Receive timeout callback, id=3001599, name=Wait node, elapsed=30, timeout=30 Set state for subtask: 3001599, operation:RETRY, state: FAILED 2024-08-09 12:34:42.885 WARN 699463 --- [pool-subtask-executor28,b1a32e67a94a4a98,5c2de9fda79f] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Receive timeout callback, id=3001599, name=Wait node, elapsed=31, timeout=30 2024-08-09 12:35:00.048 WARN 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,921ab694dd59] o.h.engine.jdbc.spi.SqlExceptionHelper : SQL Error: 0, SQLState: null 2024-08-09 12:35:00.296 ERROR 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,921ab694dd59] o.h.engine.jdbc.spi.SqlExceptionHelper : interrupt 2024-08-09 12:35:00.381 WARN 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,921ab694dd59] c.o.o.c.t.engine.runner.RunnerFactory : Execute hook failed, ex: org.springframework.transaction.CannotCreateTransactionException: Could not open JPA EntityManager for transaction; nested exception is org.hibernate.exception.GenericJDBCException: Unable to acquire JDBC Connection at org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:467) at org.springframework.transaction.support.AbstractPlatformTransactionManager.startTransaction(AbstractPlatformTransactionManager.java:400) at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373) at org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:595) at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:382) at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:119) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:137) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.data.jpa.repository.support.CrudMethodMetadataPostProcessor$CrudMethodMetadataPopulatingMethodInterceptor.invoke(CrudMethodMetadataPostProcessor.java:174) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:97) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.framewo rk.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:241) at com.sun.proxy.$Proxy383.findById(Unknown Source) at com.oceanbase.ocp.server.common.config.TaskConfiguration.lambda$setUserContext$8(TaskConfiguration.java:325) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.lambda$executeHooks$5(RunnerFactory.java:108) at java.util.ArrayList.forEach(ArrayList.java:1259) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.executeHooks(RunnerFactory.java:106) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:71) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.hibernate.exception.GenericJDBCException: Unable to acquire JDBC Connection at org.hibernate.exception.internal.StandardSQLExceptionConverter.convert(StandardSQLExceptionConverter.java:42) at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:113) at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:99) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:111) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getPhysicalConnection(LogicalConnectionManagedImpl.java:138) at org.hibernate.internal.SessionImpl.connection(SessionImpl.java:516) at org.springframework.orm.jpa.vendor.HibernateJpaDialect.beginTransaction(HibernateJpaDialect.java:152) at org.spri ngframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:421) ... 26 common frames omitted Caused by: java.sql.SQLException: interrupt at com.alibaba.druid.pool.DruidDataSource.getConnectionInternal(DruidDataSource.java:1635) at com.alibaba.druid.pool.DruidDataSource.getConnectionDirect(DruidDataSource.java:1427) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1407) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1397) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:100) at org.hibernate.engine.jdbc.connections.internal.DatasourceConnectionProviderImpl.getConnection(DatasourceConnectionProviderImpl.java:122) at org.hibernate.internal.NonContextualJdbcConnectionAccess.obtainConnection(NonContextualJdbcConnectionAccess.java:38) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:108) ... 30 common frames omitted Caused by: java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1233) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at com.alibaba.druid.pool.DruidDataSource.getConnectionInternal(DruidDataSource.java:1632) ... 37 common frames omitted 2024-08-09 12:35:03.068 INFO 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,921ab694dd59] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=3001599, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/redo, task_instance_id=3001328, task_operation=retry, ob_run_path=/home/admin/wjw_qzk/oceanbase, zone_name=zone2, wjw_qzk.1719381478.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"sys\":\"True\",\"ob_business\":\"True\"}","enable_rereplication":"True"}, ob_install_path=/home/admin/wjw_qzk/oceanbase, ob_sql_port=2881, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=OBD, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Wait node, watched_scenario=UPGRADE, sub_task_instance_id=3001599, primary_cluster_id=2, cluster_version=4.3.1.0, cluster_name=wjw_qzk, mix_rpm_names={"zone3":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm","zone2":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm","zone1":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1719381478, target_version=4.3.2.0-100000092024072912, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"],"zone2":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"],"zone1":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-08-09T12:34:10.954+08:00, ob_data_disk_path=/data}, listMap={standby_cluster_ids=[], 1719381478.wjw_qzk.zone3.server_ids=[8, 9, 10], 1719381478.wjw_qzk.zone1.server_ids=[5, 6, 7], 1719381478.wjw_qzk.zone3.host_ids=[8, 9, 10], cluster_ids=[2], primary_first_cluster_ids=[2], zone_names=[zone1, zone2, zone3], 1719381478.wjw_qzk.zone2.server_ids=[2, 3, 4] , server_ids=[2, 3, 4], standby_first_cluster_ids=[2], 1719381478.wjw_qzk.zone_names=[zone1, zone2, zone3], 1719381478.wjw_qzk.zone1.host_ids=[5, 6, 7], staged_versions=[4.3.2.0-100000092024072912], host_ids=[2, 3, 4], direct_upgrade_versions=[], 1719381478.wjw_qzk.zone2.host_ids=[2, 3, 4]}}, executor=172.22.5.124 Set state for subtask: 3001599, operation:RETRY, state: FAILED 2024-08-09 12:43:31.650 WARN 699463 --- [pool-subtask-executor26,4f3345b78e6f435e,c63c54fb4c3c] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Receive timeout callback, id=3001599, name=Wait node, elapsed=30, timeout=30 2024-08-09 12:43:50.045 WARN 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,333dd9abac8e] o.h.engine.jdbc.spi.SqlExceptionHelper : SQL Error: 0, SQLState: null 2024-08-09 12:43:50.224 ERROR 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,333dd9abac8e] o.h.engine.jdbc.spi.SqlExceptionHelper : interrupt 2024-08-09 12:43:50.399 WARN 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,333dd9abac8e] c.o.o.c.t.engine.runner.RunnerFactory : Execute hook failed, ex: org.springframework.transaction.CannotCreateTransactionException: Could not open JPA EntityManager for transaction; nested exception is org.hibernate.exception.GenericJDBCException: Unable to acquire JDBC Connection at org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:467) at org.springframework.transaction.support.AbstractPlatformTransactionManager.startTransaction(AbstractPlatformTransactionManager.java:400) at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373) at org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:595) at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:382) at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:119) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:137) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.data.jpa.repository.support.CrudMethodMetadataPostProcessor$CrudMethodMetadataPopulatingMethodInterceptor.invoke(CrudMethodMetadataPostProcessor.java:174) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:97) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.framewo rk.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:241) at com.sun.proxy.$Proxy383.findById(Unknown Source) at com.oceanbase.ocp.server.common.config.TaskConfiguration.lambda$setUserContext$8(TaskConfiguration.java:325) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.lambda$executeHooks$5(RunnerFactory.java:108) at java.util.ArrayList.forEach(ArrayList.java:1259) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.executeHooks(RunnerFactory.java:106) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:71) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.hibernate.exception.GenericJDBCException: Unable to acquire JDBC Connection at org.hibernate.exception.internal.StandardSQLExceptionConverter.convert(StandardSQLExceptionConverter.java:42) at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:113) at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:99) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:111) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getPhysicalConnection(LogicalConnectionManagedImpl.java:138) at org.hibernate.internal.SessionImpl.connection(SessionImpl.java:516) at org.springframework.orm.jpa.vendor.HibernateJpaDialect.beginTransaction(HibernateJpaDialect.java:152) at org.spri ngframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:421) ... 26 common frames omitted Caused by: java.sql.SQLException: interrupt at com.alibaba.druid.pool.DruidDataSource.getConnectionInternal(DruidDataSource.java:1635) at com.alibaba.druid.pool.DruidDataSource.getConnectionDirect(DruidDataSource.java:1427) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1407) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1397) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:100) at org.hibernate.engine.jdbc.connections.internal.DatasourceConnectionProviderImpl.getConnection(DatasourceConnectionProviderImpl.java:122) at org.hibernate.internal.NonContextualJdbcConnectionAccess.obtainConnection(NonContextualJdbcConnectionAccess.java:38) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:108) ... 30 common frames omitted Caused by: java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1233) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at com.alibaba.druid.pool.DruidDataSource.getConnectionInternal(DruidDataSource.java:1632) ... 37 common frames omitted 2024-08-09 12:43:50.815 INFO 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,333dd9abac8e] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=3001599, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/redo, task_instance_id=3001328, task_operation=retry, ob_run_path=/home/admin/wjw_qzk/oceanbase, zone_name=zone2, wjw_qzk.1719381478.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"sys\":\"True\",\"ob_business\":\"True\"}","enable_rereplication":"True"}, ob_install_path=/home/admin/wjw_qzk/oceanbase, ob_sql_port=2881, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=OBD, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Wait node, watched_scenario=UPGRADE, sub_task_instance_id=3001599, primary_cluster_id=2, cluster_version=4.3.1.0, cluster_name=wjw_qzk, mix_rpm_names={"zone3":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm","zone2":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm","zone1":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1719381478, target_version=4.3.2.0-100000092024072912, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"],"zone2":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"],"zone1":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-08-09T12:42:57.156+08:00, ob_data_disk_path=/data}, listMap={standby_cluster_ids=[], 1719381478.wjw_qzk.zone3.server_ids=[8, 9, 10], 1719381478.wjw_qzk.zone1.server_ids=[5, 6, 7], 1719381478.wjw_qzk.zone3.host_ids=[8, 9, 10], cluster_ids=[2], primary_first_cluster_ids=[2], zone_names=[zone1, zone2, zone3], 1719381478.wjw_qzk.zone2.server_ids=[2, 3, 4] , server_ids=[2, 3, 4], standby_first_cluster_ids=[2], 1719381478.wjw_qzk.zone_names=[zone1, zone2, zone3], 1719381478.wjw_qzk.zone1.host_ids=[5, 6, 7], staged_versions=[4.3.2.0-100000092024072912], host_ids=[2, 3, 4], direct_upgrade_versions=[], 1719381478.wjw_qzk.zone2.host_ids=[2, 3, 4]}}, executor=172.22.5.124 Set state for subtask: 3001599, operation:RETRY, state: FAILED 2024-08-09 12:48:21.305 WARN 699463 --- [pool-subtask-executor28,4f3345b78e6f435e,31dbbce53e43] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Receive timeout callback, id=3001599, name=Wait node, elapsed=30, timeout=30 2024-08-09 12:48:30.068 WARN 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,4e90a4d71891] o.h.engine.jdbc.spi.SqlExceptionHelper : SQL Error: 0, SQLState: null 2024-08-09 12:48:30.098 ERROR 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,4e90a4d71891] o.h.engine.jdbc.spi.SqlExceptionHelper : interrupt 2024-08-09 12:48:30.174 WARN 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,4e90a4d71891] c.o.o.c.t.engine.runner.RunnerFactory : Execute hook failed, ex: org.springframework.transaction.CannotCreateTransactionException: Could not open JPA EntityManager for transaction; nested exception is org.hibernate.exception.GenericJDBCException: Unable to acquire JDBC Connection at org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:467) at org.springframework.transaction.support.AbstractPlatformTransactionManager.startTransaction(AbstractPlatformTransactionManager.java:400) at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373) at org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:595) at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:382) at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:119) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:708) at com.oceanbase.ocp.service.iam.user.UserService$$EnhancerBySpringCGLIB$$271e2fee.findAuthenticatedUser() at com.oceanbase.ocp.server.common.config.TaskConfiguration.lambda$setUserContext$8(TaskConfiguration.java:330) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.lambda$executeHooks$5(RunnerFactory.java:108) at java.util.ArrayList.forEach(ArrayList.java:1259) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.executeHooks(RunnerFactory.java:106) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:71) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.hibernate.exception.GenericJDBCException: Unable to acquire JDBC Connection at org.hibernate.exception.internal.StandardSQLExceptionConverter.convert(StandardSQLExceptionConverter.java:42) at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:113) at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:99) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:111) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getPhysicalConnection(LogicalConnectionManagedImpl.java:138) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getConnectionForTransactionManagement(LogicalConnectionManagedImpl.java:276) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.begin(LogicalConnectionManagedImpl.java:284) at org.hibernate.resource.transaction.backend.jdbc.internal.JdbcResourceLocalTransactionCoordinatorImpl$TransactionDriverControlImpl.begin(JdbcResourceLocalTransactionCoordinatorImpl.java:246) at org.hibernate.engine.transaction.internal.TransactionImpl.begin(TransactionImpl.java:83) at org.springframework.orm.jpa.vendor.HibernateJpaDialect.beginTransaction(HibernateJpaDialect.java:164) at org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:421) ... 21 co mmon frames omitted Caused by: java.sql.SQLException: interrupt at com.alibaba.druid.pool.DruidDataSource.getConnectionInternal(DruidDataSource.java:1635) at com.alibaba.druid.pool.DruidDataSource.getConnectionDirect(DruidDataSource.java:1427) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1407) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1397) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:100) at org.hibernate.engine.jdbc.connections.internal.DatasourceConnectionProviderImpl.getConnection(DatasourceConnectionProviderImpl.java:122) at org.hibernate.internal.NonContextualJdbcConnectionAccess.obtainConnection(NonContextualJdbcConnectionAccess.java:38) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:108) ... 28 common frames omitted Caused by: java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1233) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at com.alibaba.druid.pool.DruidDataSource.getConnectionInternal(DruidDataSource.java:1632) ... 35 common frames omitted 2024-08-09 12:48:30.526 INFO 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,4e90a4d71891] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=3001599, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/redo, task_instance_id=3001328, task_operation=retry, ob_run_path=/home/admin/wjw_qzk/oceanbase, zone_name=zone2, wjw_qzk.1719381478.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"sys\":\"True\",\"ob_business\":\"True\"}","enable_rereplication":"True"}, ob_install_path=/home/admin/wjw_qzk/oceanbase, ob_sql_port=2881, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=OBD, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Wait node, watched_scenario=UPGRADE, sub_task_instance_id=3001599, primary_cluster_id=2, cluster_version=4.3.1.0, cluster_name=wjw_qzk, mix_rpm_names={"zone3":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm","zone2":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm","zone1":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1719381478, target_version=4.3.2.0-100000092024072912, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"],"zone2":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"],"zone1":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-08-09T12:47:48.768+08:00, ob_data_disk_path=/data}, listMap={standby_cluster_ids=[], 1719381478.wjw_qzk.zone3.server_ids=[8, 9, 10], 1719381478.wjw_qzk.zone1.server_ids=[5, 6, 7], 1719381478.wjw_qzk.zone3.host_ids=[8, 9, 10], cluster_ids=[2], primary_first_cluster_ids=[2], zone_names=[zone1, zone2, zone3], 1719381478.wjw_qzk.zone2.server_ids=[2, 3, 4] , server_ids=[2, 3, 4], standby_first_cluster_ids=[2], 1719381478.wjw_qzk.zone_names=[zone1, zone2, zone3], 1719381478.wjw_qzk.zone1.host_ids=[5, 6, 7], staged_versions=[4.3.2.0-100000092024072912], host_ids=[2, 3, 4], direct_upgrade_versions=[], 1719381478.wjw_qzk.zone2.host_ids=[2, 3, 4]}}, executor=172.22.5.124 Set state for subtask: 3001599, operation:RETRY, state: FAILED 2024-08-09 12:49:52.082 WARN 699463 --- [pool-subtask-executor28,4f3345b78e6f435e,31dbbce53e43] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Receive timeout callback, id=3001599, name=Wait node, elapsed=31, timeout=30 2024-08-09 12:50:04.772 WARN 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,366ebfb013cb] o.h.engine.jdbc.spi.SqlExceptionHelper : SQL Error: 0, SQLState: null 2024-08-09 12:50:07.244 ERROR 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,366ebfb013cb] o.h.engine.jdbc.spi.SqlExceptionHelper : interrupt 2024-08-09 12:50:07.409 WARN 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,366ebfb013cb] c.o.o.c.t.engine.runner.RunnerFactory : Execute hook failed, ex: org.springframework.transaction.CannotCreateTransactionException: Could not open JPA EntityManager for transaction; nested exception is org.hibernate.exception.GenericJDBCException: Unable to acquire JDBC Connection at org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:467) at org.springframework.transaction.support.AbstractPlatformTransactionManager.startTransaction(AbstractPlatformTransactionManager.java:400) at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373) at org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:595) at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:382) at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:119) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:708) at com.oceanbase.ocp.service.iam.user.UserService$$EnhancerBySpringCGLIB$$271e2fee.findAuthenticatedUser() at com.oceanbase.ocp.server.common.config.TaskConfiguration.lambda$setUserContext$8(TaskConfiguration.java:330) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.lambda$executeHooks$5(RunnerFactory.java:108) at java.util.ArrayList.forEach(ArrayList.java:1259) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.executeHooks(RunnerFactory.java:106) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:71) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:134) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.hibernate.exception.GenericJDBCException: Unable to acquire JDBC Connection at org.hibernate.exception.internal.StandardSQLExceptionConverter.convert(StandardSQLExceptionConverter.java:42) at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:113) at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:99) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:111) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getPhysicalConnection(LogicalConnectionManagedImpl.java:138) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getConnectionForTransactionManagement(LogicalConnectionManagedImpl.java:276) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.begin(LogicalConnectionManagedImpl.java:284) at org.hibernate.resource.transaction.backend.jdbc.internal.JdbcResourceLocalTransactionCoordinatorImpl$TransactionDriverControlImpl.begin(JdbcResourceLocalTransactionCoordinatorImpl.java:246) at org.hibernate.engine.transaction.internal.TransactionImpl.begin(TransactionImpl.java:83) at org.springframework.orm.jpa.vendor.HibernateJpaDialect.beginTransaction(HibernateJpaDialect.java:164) at org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:421) ... 21 co mmon frames omitted Caused by: java.sql.SQLException: interrupt at com.alibaba.druid.pool.DruidDataSource.getConnectionInternal(DruidDataSource.java:1635) at com.alibaba.druid.pool.DruidDataSource.getConnectionDirect(DruidDataSource.java:1427) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1407) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:1397) at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:100) at org.hibernate.engine.jdbc.connections.internal.DatasourceConnectionProviderImpl.getConnection(DatasourceConnectionProviderImpl.java:122) at org.hibernate.internal.NonContextualJdbcConnectionAccess.obtainConnection(NonContextualJdbcConnectionAccess.java:38) at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:108) ... 28 common frames omitted Caused by: java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1233) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at com.alibaba.druid.pool.DruidDataSource.getConnectionInternal(DruidDataSource.java:1632) ... 35 common frames omitted 2024-08-09 12:50:07.753 INFO 699463 --- [pool-manual-subtask-executor16,b49c57d96d9f4dde,366ebfb013cb] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=3001599, context=Context{parallelIdx=-1, stringMap={ob_log_disk_path=/redo, task_instance_id=3001328, task_operation=retry, ob_run_path=/home/admin/wjw_qzk/oceanbase, zone_name=zone2, wjw_qzk.1719381478.watched_parameters={"server_permanent_offline_time":"3600s","enable_rebalance":"{\"sys\":\"True\",\"ob_business\":\"True\"}","enable_rereplication":"True"}, ob_install_path=/home/admin/wjw_qzk/oceanbase, ob_sql_port=2881, cluster_id=2, ocpagent_service_name=agent, ob_disk_path_style=OBD, ob_svr_port=2882, operating_system=el7, sub_task_instance_name=Wait node, watched_scenario=UPGRADE, sub_task_instance_id=3001599, primary_cluster_id=2, cluster_version=4.3.1.0, cluster_name=wjw_qzk, mix_rpm_names={"zone3":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm","zone2":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm","zone1":"oceanbase-ce-4.3.2.0-100000092024072912.el7.aarch64.rpm"}, subtask_splitter=primary_first_cluster_ids, ob_cluster_id=1719381478, target_version=4.3.2.0-100000092024072912, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"],"zone2":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"],"zone1":["oceanbase-ce-libs-4.3.2.0-100000092024072912.el7.aarch64.rpm","oceanbase-ce-utils-4.3.2.0-100000092024072912.el7.aarch64.rpm"]}, target_cluster_status=RUNNING, ob_run_user=admin, latest_execution_start_time=2024-08-09T12:49:15.512+08:00, ob_data_disk_path=/data}, listMap={standby_cluster_ids=[], 1719381478.wjw_qzk.zone3.server_ids=[8, 9, 10], 1719381478.wjw_qzk.zone1.server_ids=[5, 6, 7], 1719381478.wjw_qzk.zone3.host_ids=[8, 9, 10], cluster_ids=[2], primary_first_cluster_ids=[2], zone_names=[zone1, zone2, zone3], 1719381478.wjw_qzk.zone2.server_ids=[2, 3, 4] , server_ids=[2, 3, 4], standby_first_cluster_ids=[2], 1719381478.wjw_qzk.zone_names=[zone1, zone2, zone3], 1719381478.wjw_qzk.zone1.host_ids=[5, 6, 7], staged_versions=[4.3.2.0-100000092024072912], host_ids=[2, 3, 4], direct_upgrade_versions=[], 1719381478.wjw_qzk.zone2.host_ids=[2, 3, 4]}}, executor=172.22.5.124