############{EXECUTE}{2022-10-21T10:24:17.331+08:00}############ 2022-10-21 10:24:17.342 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.m.t.model.SubtaskInstanceEntity : Run subtask, id=351761, context=Context{parallelIdx=-1, stringMap={cluster_name=obhuis, cluster_id=7, ocpagent_service_name=agent, prohibit_rollback=false, former_cluster_status=UNAVAILABLE, target_zone_status=RUNNING, task_instance_id=346060, task_operation=execute, target_cluster_status=RUNNING, latest_execution_start_time=2022-10-21T10:24:17.325+08:00, ob_cluster_id=7, sub_task_instance_id=351761}, listMap={7.obhuis.zone1.server_ids=[19], server_ids=[19, 20, 21], 7.obhuis.zone2.server_ids=[20], 7.obhuis.zone3.host_ids=[3], 7.obhuis.zone1.host_ids=[1], 7.obhuis.zone3.server_ids=[21], host_ids=[1, 2, 3], 7.obhuis.zone2.host_ids=[2], zone_names=[zone1, zone2, zone3]}}, executor=200.168.201.21 2022-10-21 10:24:17.402 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:24:17.469 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=obhuis, obClusterId=7, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2022-10-21 10:24:17.473 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:24:17.849 ERROR 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-21 10:24:17.855 ERROR 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-21 10:24:17.860 WARN 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:24:17.865 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:24:22.873 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:24:22.889 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=obhuis, obClusterId=null, tenantName=sys, username=root, address=200.168.201.22, port=2881, database=oceanbase) 2022-10-21 10:24:22.899 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=obhuis, obClusterId=null, tenantName=sys, username=root, address=200.168.201.22, port=2881, database=oceanbase) 2022-10-21 10:24:22.904 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alibaba.druid.pool.DruidDataSource : {dataSource-3289} closing ... 2022-10-21 10:24:22.912 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alibaba.druid.pool.DruidDataSource : {dataSource-3289} closed 2022-10-21 10:24:22.919 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=obhuis, obClusterId=null, tenantName=sys, username=root, address=200.168.201.22, port=2881, database=oceanbase) 2022-10-21 10:24:22.926 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=obhuis, obClusterId=null, tenantName=sys, username=root, address=200.168.201.22, port=2881, database=oceanbase) 2022-10-21 10:24:22.931 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alibaba.druid.pool.DruidDataSource : {dataSource-3288} closing ... 2022-10-21 10:24:22.940 ERROR 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.d.stat.DruidDataSourceStatManager : unregister mbean error javax.management.InstanceNotFoundException: com.alibaba.druid:type=DruidDataSource,id=ob-connector-datasource_direct_sys_root_200.168.201.22_2881 at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1095) ~[na:1.8.0_312] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:427) ~[na:1.8.0_312] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415) ~[na:1.8.0_312] at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546) ~[na:1.8.0_312] at com.alibaba.druid.stat.DruidDataSourceStatManager.removeDataSource(DruidDataSourceStatManager.java:202) ~[druid-1.2.8.jar!/:1.2.8] at com.alibaba.druid.pool.DruidDataSource$2.run(DruidDataSource.java:2172) [druid-1.2.8.jar!/:1.2.8] at java.security.AccessController.doPrivileged(Native Method) [na:1.8.0_312] at com.alibaba.druid.pool.DruidDataSource.unregisterMbean(DruidDataSource.java:2168) [druid-1.2.8.jar!/:1.2.8] at com.alibaba.druid.pool.DruidDataSource.close(DruidDataSource.java:2124) [druid-1.2.8.jar!/:1.2.8] at com.alibaba.druid.util.JdbcUtils.close(JdbcUtils.java:131) [druid-1.2.8.jar!/:1.2.8] at com.alipay.ocp.core.obsdk.connector.impl.AbstractConnector.close(AbstractConnector.java:110) [ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.obsdk.connector.impl.DefaultConnector.close(DefaultConnector.java:13) [ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder$1.delete(ObConnectorHolder.java:70) [ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder$1.delete(ObConnectorHolder.java:45) [ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.github.benmanes.caffeine.cache.BoundedLocalCache.l ambda$removeWithWriter$9(BoundedLocalCache.java:2147) [caffeine-2.8.0.jar!/:na] at java.util.concurrent.ConcurrentHashMap.computeIfPresent(ConcurrentHashMap.java:1769) ~[na:1.8.0_312] at com.github.benmanes.caffeine.cache.BoundedLocalCache.removeWithWriter(BoundedLocalCache.java:2137) [caffeine-2.8.0.jar!/:na] at com.github.benmanes.caffeine.cache.BoundedLocalCache.remove(BoundedLocalCache.java:2080) [caffeine-2.8.0.jar!/:na] at com.github.benmanes.caffeine.cache.LocalManualCache.invalidate(LocalManualCache.java:140) ~[caffeine-2.8.0.jar!/:na] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder.lambda$invalidate$2(ObConnectorHolder.java:154) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at java.lang.Iterable.forEach(Iterable.java:75) ~[na:1.8.0_312] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder.invalidate(ObConnectorHolder.java:151) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder.invalidate(ObConnectorHolder.java:137) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.obsdk.connector.ObConnectors.invalidate(ObConnectors.java:113) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$1(SyncClusterInfoTask.java:46) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:178) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.run(SubtaskInstanceEntity.java:221) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:26) ~[ocp-core-3.3.0-20220427 .jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:113) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:185) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:102) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:123) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] 2022-10-21 10:24:22.953 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alibaba.druid.pool.DruidDataSource : {dataSource-3288} closed 2022-10-21 10:24:22.958 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=obhuis, obClusterId=7, tenantName=sys, username=ocp_monitor, address=200.168.201.22, port=2881, database=oceanbase) 2022-10-21 10:24:22.965 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=obhuis, obClusterId=7, tenantName=sys, username=ocp_monitor, address=200.168.201.22, port=2881, database=oceanbase) 2022-10-21 10:24:22.970 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alibaba.druid.pool.DruidDataSource : {dataSource-3290} closing ... 2022-10-21 10:24:22.978 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alibaba.druid.pool.DruidDataSource : {dataSource-3290} closed 2022-10-21 10:24:22.982 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=obhuis, obClusterId=7, tenantName=sys, username=root, address=200.168.201.22, port=2881, database=oceanbase) 2022-10-21 10:24:22.990 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=obhuis, obClusterId=7, tenantName=sys, username=root, address=200.168.201.22, port=2881, database=oceanbase) 2022-10-21 10:24:22.994 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alibaba.druid.pool.DruidDataSource : {dataSource-3291} closing ... 2022-10-21 10:24:23.004 ERROR 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.d.stat.DruidDataSourceStatManager : unregister mbean error javax.management.InstanceNotFoundException: com.alibaba.druid:type=DruidDataSource,id=ob-connector-datasource_direct_sys_root_200.168.201.22_2881 at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1095) ~[na:1.8.0_312] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:427) ~[na:1.8.0_312] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415) ~[na:1.8.0_312] at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546) ~[na:1.8.0_312] at com.alibaba.druid.stat.DruidDataSourceStatManager.removeDataSource(DruidDataSourceStatManager.java:202) ~[druid-1.2.8.jar!/:1.2.8] at com.alibaba.druid.pool.DruidDataSource$2.run(DruidDataSource.java:2172) [druid-1.2.8.jar!/:1.2.8] at java.security.AccessController.doPrivileged(Native Method) [na:1.8.0_312] at com.alibaba.druid.pool.DruidDataSource.unregisterMbean(DruidDataSource.java:2168) [druid-1.2.8.jar!/:1.2.8] at com.alibaba.druid.pool.DruidDataSource.close(DruidDataSource.java:2124) [druid-1.2.8.jar!/:1.2.8] at com.alibaba.druid.util.JdbcUtils.close(JdbcUtils.java:131) [druid-1.2.8.jar!/:1.2.8] at com.alipay.ocp.core.obsdk.connector.impl.AbstractConnector.close(AbstractConnector.java:110) [ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.obsdk.connector.impl.DefaultConnector.close(DefaultConnector.java:13) [ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder$1.delete(ObConnectorHolder.java:70) [ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder$1.delete(ObConnectorHolder.java:45) [ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.github.benmanes.caffeine.cache.BoundedLocalCache.l ambda$removeWithWriter$9(BoundedLocalCache.java:2147) [caffeine-2.8.0.jar!/:na] at java.util.concurrent.ConcurrentHashMap.computeIfPresent(ConcurrentHashMap.java:1769) ~[na:1.8.0_312] at com.github.benmanes.caffeine.cache.BoundedLocalCache.removeWithWriter(BoundedLocalCache.java:2137) [caffeine-2.8.0.jar!/:na] at com.github.benmanes.caffeine.cache.BoundedLocalCache.remove(BoundedLocalCache.java:2080) [caffeine-2.8.0.jar!/:na] at com.github.benmanes.caffeine.cache.LocalManualCache.invalidate(LocalManualCache.java:140) ~[caffeine-2.8.0.jar!/:na] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder.lambda$invalidate$2(ObConnectorHolder.java:154) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at java.lang.Iterable.forEach(Iterable.java:75) ~[na:1.8.0_312] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder.invalidate(ObConnectorHolder.java:151) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder.invalidate(ObConnectorHolder.java:137) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.obsdk.connector.ObConnectors.invalidate(ObConnectors.java:113) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$1(SyncClusterInfoTask.java:46) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:178) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.run(SubtaskInstanceEntity.java:221) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:26) ~[ocp-core-3.3.0-20220427 .jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:113) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:185) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:102) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:123) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] 2022-10-21 10:24:23.018 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alibaba.druid.pool.DruidDataSource : {dataSource-3291} closed 2022-10-21 10:24:23.047 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:24:23.106 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:24:23.114 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:24:23.118 WARN 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:24:23.123 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:24:28.127 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:24:28.156 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:24:28.216 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:24:28.223 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:24:28.227 WARN 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:24:28.232 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:24:33.237 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:24:33.267 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:24:33.324 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:24:33.332 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:24:33.337 WARN 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:24:33.341 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:24:38.345 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:24:38.376 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:24:38.432 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:24:38.440 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:24:38.445 WARN 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:24:38.450 INFO 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 351761, current state: RUNNING, new state: FAILED 2022-10-21 10:24:38.455 WARN 45 --- [pool-manual-subtask-executor8,a41c4bcc314e4f20,2a6c53efce8a] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=351761, name=Sync cluster info, state=FAILED, operation=EXECUTE, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=24, startTime=2022-10-21T10:24:17.325+08:00, endTime=2022-10-21T10:24:38.454+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.run(SubtaskInstanceEntity.java:221) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:26) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:113) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:185) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:102) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:123) ~[ocp-core-3 .3.0-20220427.jar!/:3.3.0-20220427] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] ... 13 common frames omitted ############{RETRY}{2022-10-21T10:30:10.607+08:00}############ 2022-10-21 10:30:10.619 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=351761, context=Context{parallelIdx=-1, stringMap={cluster_name=obhuis, cluster_id=7, ocpagent_service_name=agent, prohibit_rollback=false, former_cluster_status=UNAVAILABLE, target_zone_status=RUNNING, task_instance_id=346060, task_operation=retry, target_cluster_status=RUNNING, latest_execution_start_time=2022-10-21T10:30:10.600+08:00, ob_cluster_id=7, sub_task_instance_id=351761}, listMap={7.obhuis.zone1.server_ids=[19], server_ids=[19, 20, 21], 7.obhuis.zone2.server_ids=[20], 7.obhuis.zone3.host_ids=[3], 7.obhuis.zone1.host_ids=[1], 7.obhuis.zone3.server_ids=[21], host_ids=[1, 2, 3], 7.obhuis.zone2.host_ids=[2], zone_names=[zone1, zone2, zone3]}}, executor=200.168.201.21 2022-10-21 10:30:10.623 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2022-10-21 10:30:10.678 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:30:10.759 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=obhuis, obClusterId=7, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2022-10-21 10:30:10.764 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:30:11.628 ERROR 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-21 10:30:11.634 ERROR 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-21 10:30:11.639 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:30:11.643 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:30:16.667 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:30:16.702 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:30:16.762 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:30:16.769 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:30:16.775 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:30:16.779 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:30:21.783 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:30:21.811 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:30:21.871 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:30:21.878 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:30:21.883 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:30:21.887 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:30:26.891 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:30:26.918 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:30:26.973 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:30:26.981 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:30:26.985 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:30:26.989 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:30:31.994 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:30:32.021 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:30:32.084 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:30:32.092 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:30:32.096 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:30:32.101 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 351761, current state: RUNNING, new state: FAILED 2022-10-21 10:30:32.107 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,7580e0746080] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=351761, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=24, startTime=2022-10-21T10:30:10.600+08:00, endTime=2022-10-21T10:30:32.105+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:233) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:113) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:185) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:102) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.tas k.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:123) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern. Retry.executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] ... 14 common frames omitted ############{RETRY}{2022-10-21T10:40:17.566+08:00}############ 2022-10-21 10:40:17.581 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=351761, context=Context{parallelIdx=-1, stringMap={cluster_name=obhuis, cluster_id=7, ocpagent_service_name=agent, prohibit_rollback=false, former_cluster_status=UNAVAILABLE, target_zone_status=RUNNING, task_instance_id=346060, task_operation=retry, target_cluster_status=RUNNING, latest_execution_start_time=2022-10-21T10:40:17.554+08:00, ob_cluster_id=7, sub_task_instance_id=351761}, listMap={7.obhuis.zone1.server_ids=[19], server_ids=[19, 20, 21], 7.obhuis.zone2.server_ids=[20], 7.obhuis.zone3.host_ids=[3], 7.obhuis.zone1.host_ids=[1], 7.obhuis.zone3.server_ids=[21], host_ids=[1, 2, 3], 7.obhuis.zone2.host_ids=[2], zone_names=[zone1, zone2, zone3]}}, executor=200.168.201.21 2022-10-21 10:40:17.586 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2022-10-21 10:40:17.645 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:40:17.781 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=obhuis, obClusterId=7, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2022-10-21 10:40:17.786 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:40:18.542 ERROR 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-21 10:40:18.548 ERROR 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-21 10:40:18.554 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:40:18.559 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:40:23.564 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:40:23.586 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=obhuis, obClusterId=7, tenantName=sys, username=ocp_monitor, address=200.168.201.22, port=2881, database=oceanbase) 2022-10-21 10:40:23.603 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=obhuis, obClusterId=7, tenantName=sys, username=ocp_monitor, address=200.168.201.22, port=2881, database=oceanbase) 2022-10-21 10:40:23.607 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alibaba.druid.pool.DruidDataSource : {dataSource-4} closing ... 2022-10-21 10:40:23.632 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alibaba.druid.pool.DruidDataSource : {dataSource-4} closed 2022-10-21 10:40:23.669 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:40:23.739 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:40:23.747 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:40:23.755 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:40:23.759 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:40:28.764 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:40:28.796 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:40:28.863 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:40:28.872 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:40:28.877 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:40:28.881 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:40:33.886 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:40:33.916 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:40:34.126 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:40:34.135 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:40:34.140 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:40:34.145 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:40:39.149 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:40:39.185 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:40:39.255 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:40:39.265 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:40:39.270 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:40:39.275 INFO 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 351761, current state: RUNNING, new state: FAILED 2022-10-21 10:40:39.282 WARN 45 --- [pool-manual-subtask-executor1,a41c4bcc314e4f20,523ae0ac51ae] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=351761, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=24, startTime=2022-10-21T10:40:17.554+08:00, endTime=2022-10-21T10:40:39.279+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:233) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:113) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:185) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:102) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.tas k.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:123) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern. Retry.executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] ... 14 common frames omitted ############{RETRY}{2022-10-21T10:43:33.599+08:00}############ 2022-10-21 10:43:33.612 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=351761, context=Context{parallelIdx=-1, stringMap={cluster_name=obhuis, cluster_id=7, ocpagent_service_name=agent, prohibit_rollback=false, former_cluster_status=UNAVAILABLE, target_zone_status=RUNNING, task_instance_id=346060, task_operation=retry, target_cluster_status=RUNNING, latest_execution_start_time=2022-10-21T10:43:33.590+08:00, ob_cluster_id=7, sub_task_instance_id=351761}, listMap={7.obhuis.zone1.server_ids=[19], server_ids=[19, 20, 21], 7.obhuis.zone2.server_ids=[20], 7.obhuis.zone3.host_ids=[3], 7.obhuis.zone1.host_ids=[1], 7.obhuis.zone3.server_ids=[21], host_ids=[1, 2, 3], 7.obhuis.zone2.host_ids=[2], zone_names=[zone1, zone2, zone3]}}, executor=200.168.201.21 2022-10-21 10:43:33.617 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2022-10-21 10:43:33.667 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:43:33.750 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=obhuis, obClusterId=7, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2022-10-21 10:43:33.754 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:43:34.469 ERROR 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-21 10:43:34.475 ERROR 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-21 10:43:34.480 WARN 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:43:34.485 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:43:39.489 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:43:39.524 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:43:39.588 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:43:39.597 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:43:39.602 WARN 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:43:39.606 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:43:44.611 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:43:44.640 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:43:44.708 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:43:44.716 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:43:44.720 WARN 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:43:44.724 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:43:49.729 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:43:49.760 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:43:49.823 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:43:49.835 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:43:49.840 WARN 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:43:49.844 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 10:43:54.850 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 10:43:54.880 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 10:43:54.941 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 10:43:54.949 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 10:43:54.954 WARN 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 10:43:54.958 INFO 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 351761, current state: RUNNING, new state: FAILED 2022-10-21 10:43:54.964 WARN 45 --- [pool-manual-subtask-executor2,a41c4bcc314e4f20,8a5c408ac7f5] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=351761, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=24, startTime=2022-10-21T10:43:33.590+08:00, endTime=2022-10-21T10:43:54.962+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:233) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:113) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:185) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:102) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.tas k.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:123) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern. Retry.executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] ... 14 common frames omitted ############{RETRY}{2022-10-21T15:35:21.243+08:00}############ 2022-10-21 15:35:21.256 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=351761, context=Context{parallelIdx=-1, stringMap={cluster_name=obhuis, cluster_id=7, ocpagent_service_name=agent, prohibit_rollback=false, former_cluster_status=UNAVAILABLE, target_zone_status=RUNNING, task_instance_id=346060, task_operation=retry, target_cluster_status=RUNNING, latest_execution_start_time=2022-10-21T15:35:21.236+08:00, ob_cluster_id=7, sub_task_instance_id=351761}, listMap={7.obhuis.zone1.server_ids=[19], server_ids=[19, 20, 21], 7.obhuis.zone2.server_ids=[20], 7.obhuis.zone3.host_ids=[3], 7.obhuis.zone1.host_ids=[1], 7.obhuis.zone3.server_ids=[21], host_ids=[1, 2, 3], 7.obhuis.zone2.host_ids=[2], zone_names=[zone1, zone2, zone3]}}, executor=200.168.201.21 2022-10-21 15:35:21.261 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2022-10-21 15:35:21.310 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 15:35:21.388 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=obhuis, obClusterId=7, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2022-10-21 15:35:21.392 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 15:35:21.837 ERROR 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-21 15:35:21.842 ERROR 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-21 15:35:21.846 WARN 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 15:35:21.850 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 15:35:26.854 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 15:35:26.886 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 15:35:26.946 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 15:35:26.954 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 15:35:26.959 WARN 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 15:35:26.964 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 15:35:31.969 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 15:35:31.998 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 15:35:32.055 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 15:35:32.063 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 15:35:32.067 WARN 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 15:35:32.071 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 15:35:37.076 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 15:35:37.103 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 15:35:37.161 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 15:35:37.169 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 15:35:37.174 WARN 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 15:35:37.178 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-21 15:35:42.182 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-21 15:35:42.212 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-21 15:35:42.272 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-21 15:35:42.281 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-21 15:35:42.285 WARN 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-21 15:35:42.289 INFO 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 351761, current state: RUNNING, new state: FAILED 2022-10-21 15:35:42.295 WARN 45 --- [pool-manual-subtask-executor3,a41c4bcc314e4f20,c597e930a55a] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=351761, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=24, startTime=2022-10-21T15:35:21.236+08:00, endTime=2022-10-21T15:35:42.294+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:233) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:113) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:185) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:102) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.tas k.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:123) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern. Retry.executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] ... 14 common frames omitted ############{RETRY}{2022-10-24T09:05:03.277+08:00}############ 2022-10-24 09:05:03.290 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=351761, context=Context{parallelIdx=-1, stringMap={cluster_name=obhuis, cluster_id=7, ocpagent_service_name=agent, prohibit_rollback=false, former_cluster_status=UNAVAILABLE, target_zone_status=RUNNING, task_instance_id=346060, task_operation=retry, target_cluster_status=RUNNING, latest_execution_start_time=2022-10-24T09:05:03.270+08:00, ob_cluster_id=7, sub_task_instance_id=351761}, listMap={7.obhuis.zone1.server_ids=[19], server_ids=[19, 20, 21], 7.obhuis.zone2.server_ids=[20], 7.obhuis.zone3.host_ids=[3], 7.obhuis.zone1.host_ids=[1], 7.obhuis.zone3.server_ids=[21], host_ids=[1, 2, 3], 7.obhuis.zone2.host_ids=[2], zone_names=[zone1, zone2, zone3]}}, executor=200.168.201.21 2022-10-24 09:05:03.295 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2022-10-24 09:05:03.347 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-24 09:05:03.421 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=obhuis, obClusterId=7, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2022-10-24 09:05:03.426 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-24 09:05:03.609 ERROR 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-24 09:05:03.614 ERROR 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2022-10-24 09:05:03.619 WARN 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-24 09:05:03.623 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-24 09:05:08.627 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-24 09:05:08.656 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-24 09:05:08.722 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-24 09:05:08.729 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-24 09:05:08.734 WARN 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-24 09:05:08.738 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-24 09:05:13.742 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-24 09:05:13.770 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-24 09:05:13.832 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-24 09:05:13.839 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-24 09:05:13.843 WARN 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-24 09:05:13.848 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-24 09:05:18.853 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-24 09:05:18.882 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-24 09:05:18.941 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-24 09:05:18.948 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-24 09:05:18.952 WARN 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-24 09:05:18.957 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2022-10-24 09:05:23.961 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=obhuis 2022-10-24 09:05:23.994 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=7, clusterName=obhuis, obClusterId=7 2022-10-24 09:05:24.058 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obhuis, tenantName=sys, dbUser=ocp_monitor 2022-10-24 09:05:24.065 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=obhuis, obClusterId=7, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2022-10-24 09:05:24.071 WARN 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor 2022-10-24 09:05:24.076 INFO 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 351761, current state: RUNNING, new state: FAILED 2022-10-24 09:05:24.083 WARN 45 --- [pool-manual-subtask-executor4,a41c4bcc314e4f20,76a50eaca878] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=351761, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=24, startTime=2022-10-24T09:05:03.270+08:00, endTime=2022-10-24T09:05:24.080+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:233) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:113) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:185) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:102) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.core.tas k.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:123) ~[ocp-core-3.3.0-20220427.jar!/:3.3.0-20220427] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=obhuis,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] at com.alipay.ocp.common.pattern. Retry.executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.0-20220427.jar!/:3.3.0-20220427] ... 14 common frames omitted