############{EXECUTE}{2024-07-16T10:28:36.196+08:00}############ 2024-07-16 10:28:36.208 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.m.t.model.SubtaskInstanceEntity : Run subtask, id=1004600, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=1004480, agent_async_task_token_map={"1004596":"2e0a0539-3934-4b25-939a-fb56b2e145af"}, task_operation=execute, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, primary_zone=, cluster_id=1000002, sub_task_instance_id=1004600, cluster_version=4.3.1.0, cluster_name=OB, target_server_status=RUNNING, startup_option_string=, ob_version=4.3.1.0, service_name=OB:1721059595, target_zone_status=RUNNING, ob_cluster_id=1721059595, zone1.idc=cs, root_sys_password=******, service_type=OB_CLUSTER, rpm_name=oceanbase-4.3.1.0-100010022024052916.el7.x86_64.rpm, connection_mode=direct, target_cluster_status=RUNNING, latest_execution_start_time=2024-07-16T10:28:36.190+08:00, ob_data_disk_path=/data/1}, listMap={root_servers=[1000001], ob_connect_address_list=[172.21.34.5:2881], server_ids=[1000002], dep_rpm_names=[], host_ids=[1000001], zone1.host_ids=[1000001], zone_names=[zone1]}}, executor=172.17.0.1 2024-07-16 10:28:36.225 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:36.260 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : system obproxy is not exists or unavailable now, use direct mode instead 2024-07-16 10:28:36.264 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:36.267 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:36.273 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:36.276 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `zone`, MAX(CASE `name` WHEN 'region' THEN `info` ELSE '' END ) `region`, MAX(CASE `name` WHEN 'idc' THEN `info` ELSE '' END ) `idc`, MAX(CASE `name` WHEN 'status' THEN `info` ELSE '' END ) `status`, MAX(CASE `name` WHEN 'merge_status' THEN `info` ELSE '' END ) `merge_status`, MAX(CASE `name` WHEN 'broadcast_version' THEN `value` ELSE 0 END ) `broadcast_version`, MAX(CASE `name` WHEN 'all_merged_version' THEN `value` ELSE 0 END ) `all_merged_version`, MAX(CASE `name` WHEN 'last_merged_version' THEN `value` ELSE 0 END ) `last_merged_version`, MAX(CASE `name` WHEN 'merge_start_time' THEN `value` ELSE 0 END ) `merge_start_time`, MAX(CASE `name` WHEN 'last_merged_time' THEN `value` ELSE 0 END ) `last_merged_time`, MAX(CASE `name` WHEN 'is_merge_timeout' THEN `value` ELSE 0 END ) `merge_timeout` FROM oceanbase.__all_zone WHERE `zone` <> '' GROUP BY `zone` 2024-07-16 10:28:36.281 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:36.285 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT zone, gmt_create, gmt_modified, svr_ip, svr_port, inner_port, with_rootserver, UPPER(`status`) as `status`, build_version, stop_time, start_service_time, last_offline_time, block_migrate_in_time, id FROM oceanbase.__all_server ORDER BY zone ASC, id ASC 2024-07-16 10:28:36.294 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.alipay.ocp.core.obsdk.util.ObSdkUtils : check observer active, observer=172.21.34.5:2882, status=ACTIVE, startServiceTime=1721096913130430, active=true 2024-07-16 10:28:36.298 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.alipay.ocp.core.obsdk.util.ObSdkUtils : check observer stopped, observer=172.21.34.5:2882, stopTime=0, stopped=false 2024-07-16 10:28:36.302 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : get current info from ob cluster, clusterId=1000002 2024-07-16 10:28:36.306 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : detect zone changed, before=ObZoneEntity{id=1000002, name=zone1, idcId=null, status=CREATING, innerStatus=null, compactionStatus=null, startTime=null, stopTime=null}, current=ObZoneEntity{id=null, name=zone1, idcId=1, status=RUNNING, innerStatus=ACTIVE, compactionStatus=null, startTime=null, stopTime=null} 2024-07-16 10:28:36.324 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : zone updated, zoneName=zone1 2024-07-16 10:28:36.328 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : detect server changed, before=ObServerEntity{id=1000002, ip=172.21.34.5, port=2882, sqlPort=2881, version=null, withRootserver=true, hostId=1000001, status=CREATING, innerStatus=null, startTime=null, stopTime=null}, current=ObServerEntity{id=null, ip=172.21.34.5, port=2882, sqlPort=2881, version=4.3.1.0_100010022024052916-9a755dd27aaaf5e34e8ad6ad95942a0d89e7c80f(May 29 2024 17:21:45), withRootserver=true, hostId=1000001, status=RUNNING, innerStatus=ACTIVE, startTime=2024-07-16T02:28:33.130430Z, stopTime=1970-01-01T00:00Z} 2024-07-16 10:28:36.348 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : update zones and servers done, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:36.350 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:36.367 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:28:36.373 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : system obproxy is not exists or unavailable now, use direct mode instead 2024-07-16 10:28:36.377 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:36.381 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:36.387 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:36.391 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:36.397 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:36.401 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster 2024-07-16 10:28:36.407 ERROR 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] queryForObject failed, sql:[SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster], error message:[StatementCallback; bad SQL grammar [SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster]; nested exception is java.sql.SQLSyntaxErrorException: (conn=3221487617) Table 'oceanbase.v$ob_cluster' doesn't exist] 2024-07-16 10:28:36.411 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:(conn=3221487617) Table 'oceanbase.v$ob_cluster' doesn't exist 2024-07-16 10:28:36.416 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:28:41.419 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:28:41.423 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=null, tenantName=sys, username=root, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:41.427 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=null, tenantName=sys, username=root, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:41.430 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-22} closing ... 2024-07-16 10:28:41.445 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-22} closed 2024-07-16 10:28:41.449 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:41.452 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:41.455 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-24} closing ... 2024-07-16 10:28:41.461 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-24} closed 2024-07-16 10:28:41.465 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=null, tenantName=sys, username=root, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:41.467 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=null, tenantName=sys, username=root, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:41.469 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-23} closing ... 2024-07-16 10:28:41.474 ERROR 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.d.stat.DruidDataSourceStatManager : unregister mbean error javax.management.InstanceNotFoundException: com.alibaba.druid:type=DruidDataSource,id=ob-connector-datasource_direct_sys_root_172.21.34.5_2881 at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1095) ~[na:1.8.0_312] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:427) ~[na:1.8.0_312] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415) ~[na:1.8.0_312] at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546) ~[na:1.8.0_312] at com.alibaba.druid.stat.DruidDataSourceStatManager.removeDataSource(DruidDataSourceStatManager.java:202) ~[druid-1.2.8.jar!/:1.2.8] at com.alibaba.druid.pool.DruidDataSource$2.run(DruidDataSource.java:2172) [druid-1.2.8.jar!/:1.2.8] at java.security.AccessController.doPrivileged(Native Method) [na:1.8.0_312] at com.alibaba.druid.pool.DruidDataSource.unregisterMbean(DruidDataSource.java:2168) [druid-1.2.8.jar!/:1.2.8] at com.alibaba.druid.pool.DruidDataSource.close(DruidDataSource.java:2124) [druid-1.2.8.jar!/:1.2.8] at com.alibaba.druid.util.JdbcUtils.close(JdbcUtils.java:131) [druid-1.2.8.jar!/:1.2.8] at com.alipay.ocp.core.obsdk.connector.impl.AbstractConnector.close(AbstractConnector.java:110) [ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.obsdk.connector.impl.DefaultConnector.close(DefaultConnector.java:13) [ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder$1.delete(ObConnectorHolder.java:70) [ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder$1.delete(ObConnectorHolder.java:45) [ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.github.benmanes.caffeine.cache.BoundedLocalCache.lamb da$removeWithWriter$9(BoundedLocalCache.java:2147) [caffeine-2.8.0.jar!/:na] at java.util.concurrent.ConcurrentHashMap.computeIfPresent(ConcurrentHashMap.java:1769) ~[na:1.8.0_312] at com.github.benmanes.caffeine.cache.BoundedLocalCache.removeWithWriter(BoundedLocalCache.java:2137) [caffeine-2.8.0.jar!/:na] at com.github.benmanes.caffeine.cache.BoundedLocalCache.remove(BoundedLocalCache.java:2080) [caffeine-2.8.0.jar!/:na] at com.github.benmanes.caffeine.cache.LocalManualCache.invalidate(LocalManualCache.java:140) ~[caffeine-2.8.0.jar!/:na] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder.lambda$invalidate$2(ObConnectorHolder.java:154) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.lang.Iterable.forEach(Iterable.java:75) ~[na:1.8.0_312] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder.invalidate(ObConnectorHolder.java:151) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.obsdk.connector.ObConnectorHolder.invalidate(ObConnectorHolder.java:137) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.obsdk.connector.ObConnectors.invalidate(ObConnectors.java:113) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$1(SyncClusterInfoTask.java:46) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:178) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.run(SubtaskInstanceEntity.java:225) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:26) ~[ocp-core-3.3.4-20221201.ja r!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] 2024-07-16 10:28:41.482 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-23} closed 2024-07-16 10:28:41.494 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:41.519 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:28:41.523 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : system obproxy is not exists or unavailable now, use direct mode instead 2024-07-16 10:28:41.527 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:41.530 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=null, port=null, obsAddrList=[ObServerAddr(address=172.21.34.5, port=2881)], username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:28:41.543 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector success 2024-07-16 10:28:41.549 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:41.552 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:41.557 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:41.562 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `zone`, MAX(CASE `name` WHEN 'region' THEN `info` ELSE '' END ) `region`, MAX(CASE `name` WHEN 'idc' THEN `info` ELSE '' END ) `idc`, MAX(CASE `name` WHEN 'status' THEN `info` ELSE '' END ) `status`, MAX(CASE `name` WHEN 'merge_status' THEN `info` ELSE '' END ) `merge_status`, MAX(CASE `name` WHEN 'broadcast_version' THEN `value` ELSE 0 END ) `broadcast_version`, MAX(CASE `name` WHEN 'all_merged_version' THEN `value` ELSE 0 END ) `all_merged_version`, MAX(CASE `name` WHEN 'last_merged_version' THEN `value` ELSE 0 END ) `last_merged_version`, MAX(CASE `name` WHEN 'merge_start_time' THEN `value` ELSE 0 END ) `merge_start_time`, MAX(CASE `name` WHEN 'last_merged_time' THEN `value` ELSE 0 END ) `last_merged_time`, MAX(CASE `name` WHEN 'is_merge_timeout' THEN `value` ELSE 0 END ) `merge_timeout` FROM oceanbase.__all_zone WHERE `zone` <> '' GROUP BY `zone` 2024-07-16 10:28:41.568 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:41.572 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT zone, gmt_create, gmt_modified, svr_ip, svr_port, inner_port, with_rootserver, UPPER(`status`) as `status`, build_version, stop_time, start_service_time, last_offline_time, block_migrate_in_time, id FROM oceanbase.__all_server ORDER BY zone ASC, id ASC 2024-07-16 10:28:41.577 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.alipay.ocp.core.obsdk.util.ObSdkUtils : check observer active, observer=172.21.34.5:2882, status=ACTIVE, startServiceTime=1721096913130430, active=true 2024-07-16 10:28:41.579 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.alipay.ocp.core.obsdk.util.ObSdkUtils : check observer stopped, observer=172.21.34.5:2882, stopTime=0, stopped=false 2024-07-16 10:28:41.581 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : get current info from ob cluster, clusterId=1000002 2024-07-16 10:28:41.583 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : detect zone changed, before=ObZoneEntity{id=1000002, name=zone1, idcId=1, status=RUNNING, innerStatus=ACTIVE, compactionStatus=null, startTime=null, stopTime=null}, current=ObZoneEntity{id=null, name=zone1, idcId=1, status=RUNNING, innerStatus=ACTIVE, compactionStatus=null, startTime=null, stopTime=null} 2024-07-16 10:28:41.593 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : zone updated, zoneName=zone1 2024-07-16 10:28:41.596 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : detect server changed, before=ObServerEntity{id=1000002, ip=172.21.34.5, port=2882, sqlPort=2881, version=4.3.1.0_100010022024052916-9a755dd27aaaf5e34e8ad6ad95942a0d89e7c80f(May 29 2024 17:21:45), withRootserver=true, hostId=1000001, status=RUNNING, innerStatus=ACTIVE, startTime=2024-07-16T10:28:33+08:00, stopTime=1970-01-01T08:00+08:00}, current=ObServerEntity{id=null, ip=172.21.34.5, port=2882, sqlPort=2881, version=4.3.1.0_100010022024052916-9a755dd27aaaf5e34e8ad6ad95942a0d89e7c80f(May 29 2024 17:21:45), withRootserver=true, hostId=1000001, status=RUNNING, innerStatus=ACTIVE, startTime=2024-07-16T02:28:33.130430Z, stopTime=1970-01-01T00:00Z} 2024-07-16 10:28:41.609 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : update zones and servers done, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:41.612 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:41.627 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:28:41.630 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : system obproxy is not exists or unavailable now, use direct mode instead 2024-07-16 10:28:41.633 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:41.636 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:41.640 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:41.645 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:41.650 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:41.656 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster 2024-07-16 10:28:41.662 ERROR 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] queryForObject failed, sql:[SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster], error message:[StatementCallback; bad SQL grammar [SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster]; nested exception is java.sql.SQLSyntaxErrorException: (conn=3221487618) Table 'oceanbase.v$ob_cluster' doesn't exist] 2024-07-16 10:28:41.666 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:(conn=3221487618) Table 'oceanbase.v$ob_cluster' doesn't exist 2024-07-16 10:28:41.669 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:28:46.672 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:28:46.676 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:46.679 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:46.682 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-25} closing ... 2024-07-16 10:28:46.691 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-25} closed 2024-07-16 10:28:46.711 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:46.746 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:28:46.751 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : system obproxy is not exists or unavailable now, use direct mode instead 2024-07-16 10:28:46.755 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:46.758 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=null, port=null, obsAddrList=[ObServerAddr(address=172.21.34.5, port=2881)], username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:28:46.770 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector success 2024-07-16 10:28:46.775 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:46.781 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:46.787 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:46.793 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `zone`, MAX(CASE `name` WHEN 'region' THEN `info` ELSE '' END ) `region`, MAX(CASE `name` WHEN 'idc' THEN `info` ELSE '' END ) `idc`, MAX(CASE `name` WHEN 'status' THEN `info` ELSE '' END ) `status`, MAX(CASE `name` WHEN 'merge_status' THEN `info` ELSE '' END ) `merge_status`, MAX(CASE `name` WHEN 'broadcast_version' THEN `value` ELSE 0 END ) `broadcast_version`, MAX(CASE `name` WHEN 'all_merged_version' THEN `value` ELSE 0 END ) `all_merged_version`, MAX(CASE `name` WHEN 'last_merged_version' THEN `value` ELSE 0 END ) `last_merged_version`, MAX(CASE `name` WHEN 'merge_start_time' THEN `value` ELSE 0 END ) `merge_start_time`, MAX(CASE `name` WHEN 'last_merged_time' THEN `value` ELSE 0 END ) `last_merged_time`, MAX(CASE `name` WHEN 'is_merge_timeout' THEN `value` ELSE 0 END ) `merge_timeout` FROM oceanbase.__all_zone WHERE `zone` <> '' GROUP BY `zone` 2024-07-16 10:28:46.798 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:46.803 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT zone, gmt_create, gmt_modified, svr_ip, svr_port, inner_port, with_rootserver, UPPER(`status`) as `status`, build_version, stop_time, start_service_time, last_offline_time, block_migrate_in_time, id FROM oceanbase.__all_server ORDER BY zone ASC, id ASC 2024-07-16 10:28:46.808 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.alipay.ocp.core.obsdk.util.ObSdkUtils : check observer active, observer=172.21.34.5:2882, status=ACTIVE, startServiceTime=1721096913130430, active=true 2024-07-16 10:28:46.812 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.alipay.ocp.core.obsdk.util.ObSdkUtils : check observer stopped, observer=172.21.34.5:2882, stopTime=0, stopped=false 2024-07-16 10:28:46.815 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : get current info from ob cluster, clusterId=1000002 2024-07-16 10:28:46.817 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : detect zone changed, before=ObZoneEntity{id=1000002, name=zone1, idcId=1, status=RUNNING, innerStatus=ACTIVE, compactionStatus=null, startTime=null, stopTime=null}, current=ObZoneEntity{id=null, name=zone1, idcId=1, status=RUNNING, innerStatus=ACTIVE, compactionStatus=null, startTime=null, stopTime=null} 2024-07-16 10:28:46.831 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : zone updated, zoneName=zone1 2024-07-16 10:28:46.833 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : detect server changed, before=ObServerEntity{id=1000002, ip=172.21.34.5, port=2882, sqlPort=2881, version=4.3.1.0_100010022024052916-9a755dd27aaaf5e34e8ad6ad95942a0d89e7c80f(May 29 2024 17:21:45), withRootserver=true, hostId=1000001, status=RUNNING, innerStatus=ACTIVE, startTime=2024-07-16T10:28:33+08:00, stopTime=1970-01-01T08:00+08:00}, current=ObServerEntity{id=null, ip=172.21.34.5, port=2882, sqlPort=2881, version=4.3.1.0_100010022024052916-9a755dd27aaaf5e34e8ad6ad95942a0d89e7c80f(May 29 2024 17:21:45), withRootserver=true, hostId=1000001, status=RUNNING, innerStatus=ACTIVE, startTime=2024-07-16T02:28:33.130430Z, stopTime=1970-01-01T00:00Z} 2024-07-16 10:28:46.849 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : update zones and servers done, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:46.853 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:46.867 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:28:46.871 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : system obproxy is not exists or unavailable now, use direct mode instead 2024-07-16 10:28:46.874 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:46.878 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:46.882 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:46.886 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:46.891 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:46.895 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster 2024-07-16 10:28:46.900 ERROR 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] queryForObject failed, sql:[SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster], error message:[StatementCallback; bad SQL grammar [SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster]; nested exception is java.sql.SQLSyntaxErrorException: (conn=3221487619) Table 'oceanbase.v$ob_cluster' doesn't exist] 2024-07-16 10:28:46.904 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:(conn=3221487619) Table 'oceanbase.v$ob_cluster' doesn't exist 2024-07-16 10:28:46.907 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:28:51.910 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:28:51.914 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:51.918 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:51.922 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-26} closing ... 2024-07-16 10:28:51.934 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-26} closed 2024-07-16 10:28:51.961 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:51.983 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:28:51.987 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : system obproxy is not exists or unavailable now, use direct mode instead 2024-07-16 10:28:51.989 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:51.991 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=null, port=null, obsAddrList=[ObServerAddr(address=172.21.34.5, port=2881)], username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:28:52.002 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector success 2024-07-16 10:28:52.012 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:52.016 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:52.021 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:52.027 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `zone`, MAX(CASE `name` WHEN 'region' THEN `info` ELSE '' END ) `region`, MAX(CASE `name` WHEN 'idc' THEN `info` ELSE '' END ) `idc`, MAX(CASE `name` WHEN 'status' THEN `info` ELSE '' END ) `status`, MAX(CASE `name` WHEN 'merge_status' THEN `info` ELSE '' END ) `merge_status`, MAX(CASE `name` WHEN 'broadcast_version' THEN `value` ELSE 0 END ) `broadcast_version`, MAX(CASE `name` WHEN 'all_merged_version' THEN `value` ELSE 0 END ) `all_merged_version`, MAX(CASE `name` WHEN 'last_merged_version' THEN `value` ELSE 0 END ) `last_merged_version`, MAX(CASE `name` WHEN 'merge_start_time' THEN `value` ELSE 0 END ) `merge_start_time`, MAX(CASE `name` WHEN 'last_merged_time' THEN `value` ELSE 0 END ) `last_merged_time`, MAX(CASE `name` WHEN 'is_merge_timeout' THEN `value` ELSE 0 END ) `merge_timeout` FROM oceanbase.__all_zone WHERE `zone` <> '' GROUP BY `zone` 2024-07-16 10:28:52.034 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:52.039 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT zone, gmt_create, gmt_modified, svr_ip, svr_port, inner_port, with_rootserver, UPPER(`status`) as `status`, build_version, stop_time, start_service_time, last_offline_time, block_migrate_in_time, id FROM oceanbase.__all_server ORDER BY zone ASC, id ASC 2024-07-16 10:28:52.044 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.alipay.ocp.core.obsdk.util.ObSdkUtils : check observer active, observer=172.21.34.5:2882, status=ACTIVE, startServiceTime=1721096913130430, active=true 2024-07-16 10:28:52.047 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.alipay.ocp.core.obsdk.util.ObSdkUtils : check observer stopped, observer=172.21.34.5:2882, stopTime=0, stopped=false 2024-07-16 10:28:52.051 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : get current info from ob cluster, clusterId=1000002 2024-07-16 10:28:52.054 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : detect zone changed, before=ObZoneEntity{id=1000002, name=zone1, idcId=1, status=RUNNING, innerStatus=ACTIVE, compactionStatus=null, startTime=null, stopTime=null}, current=ObZoneEntity{id=null, name=zone1, idcId=1, status=RUNNING, innerStatus=ACTIVE, compactionStatus=null, startTime=null, stopTime=null} 2024-07-16 10:28:52.067 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : zone updated, zoneName=zone1 2024-07-16 10:28:52.070 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : detect server changed, before=ObServerEntity{id=1000002, ip=172.21.34.5, port=2882, sqlPort=2881, version=4.3.1.0_100010022024052916-9a755dd27aaaf5e34e8ad6ad95942a0d89e7c80f(May 29 2024 17:21:45), withRootserver=true, hostId=1000001, status=RUNNING, innerStatus=ACTIVE, startTime=2024-07-16T10:28:33+08:00, stopTime=1970-01-01T08:00+08:00}, current=ObServerEntity{id=null, ip=172.21.34.5, port=2882, sqlPort=2881, version=4.3.1.0_100010022024052916-9a755dd27aaaf5e34e8ad6ad95942a0d89e7c80f(May 29 2024 17:21:45), withRootserver=true, hostId=1000001, status=RUNNING, innerStatus=ACTIVE, startTime=2024-07-16T02:28:33.130430Z, stopTime=1970-01-01T00:00Z} 2024-07-16 10:28:52.085 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : update zones and servers done, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:52.090 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:52.111 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:28:52.116 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : system obproxy is not exists or unavailable now, use direct mode instead 2024-07-16 10:28:52.119 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:52.123 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:52.127 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:52.130 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:52.137 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:52.141 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster 2024-07-16 10:28:52.146 ERROR 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] queryForObject failed, sql:[SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster], error message:[StatementCallback; bad SQL grammar [SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster]; nested exception is java.sql.SQLSyntaxErrorException: (conn=3221487620) Table 'oceanbase.v$ob_cluster' doesn't exist] 2024-07-16 10:28:52.150 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:(conn=3221487620) Table 'oceanbase.v$ob_cluster' doesn't exist 2024-07-16 10:28:52.153 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:28:57.159 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:28:57.165 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:57.169 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:57.174 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-27} closing ... 2024-07-16 10:28:57.185 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alibaba.druid.pool.DruidDataSource : {dataSource-27} closed 2024-07-16 10:28:57.205 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:57.233 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:28:57.237 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : system obproxy is not exists or unavailable now, use direct mode instead 2024-07-16 10:28:57.239 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:28:57.241 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=direct, address=null, port=null, obsAddrList=[ObServerAddr(address=172.21.34.5, port=2881)], username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:28:57.250 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector success 2024-07-16 10:28:57.256 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:57.260 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:57.265 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:57.269 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT `zone`, MAX(CASE `name` WHEN 'region' THEN `info` ELSE '' END ) `region`, MAX(CASE `name` WHEN 'idc' THEN `info` ELSE '' END ) `idc`, MAX(CASE `name` WHEN 'status' THEN `info` ELSE '' END ) `status`, MAX(CASE `name` WHEN 'merge_status' THEN `info` ELSE '' END ) `merge_status`, MAX(CASE `name` WHEN 'broadcast_version' THEN `value` ELSE 0 END ) `broadcast_version`, MAX(CASE `name` WHEN 'all_merged_version' THEN `value` ELSE 0 END ) `all_merged_version`, MAX(CASE `name` WHEN 'last_merged_version' THEN `value` ELSE 0 END ) `last_merged_version`, MAX(CASE `name` WHEN 'merge_start_time' THEN `value` ELSE 0 END ) `merge_start_time`, MAX(CASE `name` WHEN 'last_merged_time' THEN `value` ELSE 0 END ) `last_merged_time`, MAX(CASE `name` WHEN 'is_merge_timeout' THEN `value` ELSE 0 END ) `merge_timeout` FROM oceanbase.__all_zone WHERE `zone` <> '' GROUP BY `zone` 2024-07-16 10:28:57.274 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:57.278 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT zone, gmt_create, gmt_modified, svr_ip, svr_port, inner_port, with_rootserver, UPPER(`status`) as `status`, build_version, stop_time, start_service_time, last_offline_time, block_migrate_in_time, id FROM oceanbase.__all_server ORDER BY zone ASC, id ASC 2024-07-16 10:28:57.283 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.alipay.ocp.core.obsdk.util.ObSdkUtils : check observer active, observer=172.21.34.5:2882, status=ACTIVE, startServiceTime=1721096913130430, active=true 2024-07-16 10:28:57.286 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.alipay.ocp.core.obsdk.util.ObSdkUtils : check observer stopped, observer=172.21.34.5:2882, stopTime=0, stopped=false 2024-07-16 10:28:57.288 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : get current info from ob cluster, clusterId=1000002 2024-07-16 10:28:57.291 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : detect zone changed, before=ObZoneEntity{id=1000002, name=zone1, idcId=1, status=RUNNING, innerStatus=ACTIVE, compactionStatus=null, startTime=null, stopTime=null}, current=ObZoneEntity{id=null, name=zone1, idcId=1, status=RUNNING, innerStatus=ACTIVE, compactionStatus=null, startTime=null, stopTime=null} 2024-07-16 10:28:57.304 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : zone updated, zoneName=zone1 2024-07-16 10:28:57.307 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : detect server changed, before=ObServerEntity{id=1000002, ip=172.21.34.5, port=2882, sqlPort=2881, version=4.3.1.0_100010022024052916-9a755dd27aaaf5e34e8ad6ad95942a0d89e7c80f(May 29 2024 17:21:45), withRootserver=true, hostId=1000001, status=RUNNING, innerStatus=ACTIVE, startTime=2024-07-16T10:28:33+08:00, stopTime=1970-01-01T08:00+08:00}, current=ObServerEntity{id=null, ip=172.21.34.5, port=2882, sqlPort=2881, version=4.3.1.0_100010022024052916-9a755dd27aaaf5e34e8ad6ad95942a0d89e7c80f(May 29 2024 17:21:45), withRootserver=true, hostId=1000001, status=RUNNING, innerStatus=ACTIVE, startTime=2024-07-16T02:28:33.130430Z, stopTime=1970-01-01T00:00Z} 2024-07-16 10:28:57.318 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : update zones and servers done, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:57.320 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:28:57.331 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:28:57.334 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.s.o.o.f.ConnectPropertiesBuilder : system obproxy is not exists or unavailable now, use direct mode instead 2024-07-16 10:28:57.338 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:57.341 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:57.348 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:57.353 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: select /** QUERY_TIMEOUT(60000000) */ max(value) value from oceanbase.__all_virtual_sys_parameter_stat where name = 'min_observer_version' 2024-07-16 10:28:57.360 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-07-16 10:28:57.364 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster 2024-07-16 10:28:57.368 ERROR 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] queryForObject failed, sql:[SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster], error message:[StatementCallback; bad SQL grammar [SELECT cluster_id, cluster_name, created, cluster_role, cluster_status, current_scn, standby_became_primary_scn, primary_cluster_id, (time_to_usec(now(6)) - current_scn)/1000000 as delay_time, protection_mode, redo_transport_options, protection_level FROM oceanbase.v$ob_cluster]; nested exception is java.sql.SQLSyntaxErrorException: (conn=3221487621) Table 'oceanbase.v$ob_cluster' doesn't exist] 2024-07-16 10:28:57.371 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:(conn=3221487621) Table 'oceanbase.v$ob_cluster' doesn't exist 2024-07-16 10:28:57.374 INFO 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 1004600, current state: RUNNING, new state: FAILED 2024-07-16 10:28:57.378 WARN 50 --- [pool-manual-subtask-executor5,23edc2ef01174bb9,57202dd2f72a] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=1004600, name=Sync cluster info, state=FAILED, operation=EXECUTE, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=20, startTime=2024-07-16T10:28:36.190+08:00, endTime=2024-07-16T10:28:57.376+08:00} java.lang.RuntimeException: com.alipay.ocp.core.obsdk.exception.OceanBaseException: (conn=3221487621) Table 'oceanbase.v$ob_cluster' doesn't exist at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.run(SubtaskInstanceEntity.java:225) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:26) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.con current.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.obsdk.exception.OceanBaseException: (conn=3221487621) Table 'oceanbase.v$ob_cluster' doesn't exist at com.alipay.ocp.core.obsdk.connector.ConnectTemplate.customizeException(ConnectTemplate.java:377) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.obsdk.connector.ConnectTemplate.customizeException(ConnectTemplate.java:369) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.obsdk.connector.ConnectTemplate.queryForObject(ConnectTemplate.java:124) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.obsdk.operator.cluster.MysqlClusterOperator.getCluster(MysqlClusterOperator.java:324) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.updateCluster(ClusterSyncService.java:324) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:248) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] ... 13 common frames omitted Caused by: java.sql.SQLSyntaxErrorExc eption: (conn=3221487621) Table 'oceanbase.v$ob_cluster' doesn't exist at com.oceanbase.jdbc.internal.util.exceptions.ExceptionFactory.createException(ExceptionFactory.java:108) ~[oceanbase-client-2.2.7.jar!/:na] at com.oceanbase.jdbc.internal.util.exceptions.ExceptionFactory.create(ExceptionFactory.java:200) ~[oceanbase-client-2.2.7.jar!/:na] at com.oceanbase.jdbc.OceanBaseStatement.executeExceptionEpilogue(OceanBaseStatement.java:304) ~[oceanbase-client-2.2.7.jar!/:na] at com.oceanbase.jdbc.OceanBaseStatement.executeInternal(OceanBaseStatement.java:393) ~[oceanbase-client-2.2.7.jar!/:na] at com.oceanbase.jdbc.OceanBaseStatement.executeQuery(OceanBaseStatement.java:650) ~[oceanbase-client-2.2.7.jar!/:na] at com.alibaba.druid.pool.DruidPooledStatement.executeQuery(DruidPooledStatement.java:308) ~[druid-1.2.8.jar!/:1.2.8] at org.springframework.jdbc.core.JdbcTemplate$1QueryStatementCallback.doInStatement(JdbcTemplate.java:439) ~[spring-jdbc-5.2.4.RELEASE.jar!/:5.2.4.RELEASE] at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:376) ~[spring-jdbc-5.2.4.RELEASE.jar!/:5.2.4.RELEASE] at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:452) ~[spring-jdbc-5.2.4.RELEASE.jar!/:5.2.4.RELEASE] at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:462) ~[spring-jdbc-5.2.4.RELEASE.jar!/:5.2.4.RELEASE] at org.springframework.jdbc.core.JdbcTemplate.queryForObject(JdbcTemplate.java:473) ~[spring-jdbc-5.2.4.RELEASE.jar!/:5.2.4.RELEASE] at com.alipay.ocp.core.obsdk.connector.ConnectTemplate.queryForObject(ConnectTemplate.java:121) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] ... 20 common frames omitted Caused by: com.oceanbase.jdbc.internal.util.exceptions.OceanBaseSqlException: Table 'oceanbase.v$ob_cluster' doesn't exist at com.oceanbase.jdbc.internal.util.exceptions.OceanBaseSqlException.of(OceanBaseSqlException.java:80) ~[oceanbase-client-2.2.7.jar!/:na] at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.exceptionWithQuery(AbstractQueryProtocol. java:196) ~[oceanbase-client-2.2.7.jar!/:na] at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.executeQuery(AbstractQueryProtocol.java:304) ~[oceanbase-client-2.2.7.jar!/:na] at com.oceanbase.jdbc.OceanBaseStatement.executeInternal(OceanBaseStatement.java:387) ~[oceanbase-client-2.2.7.jar!/:na] ... 28 common frames omitted Caused by: java.sql.SQLException: Table 'oceanbase.v$ob_cluster' doesn't exist at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.readErrorPacket(AbstractQueryProtocol.java:1975) ~[oceanbase-client-2.2.7.jar!/:na] at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.readPacket(AbstractQueryProtocol.java:1840) ~[oceanbase-client-2.2.7.jar!/:na] at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.getResult(AbstractQueryProtocol.java:1730) ~[oceanbase-client-2.2.7.jar!/:na] at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.executeQuery(AbstractQueryProtocol.java:298) ~[oceanbase-client-2.2.7.jar!/:na] ... 29 common frames omitted ############{RETRY}{2024-07-16T10:34:06.518+08:00}############ 2024-07-16 10:34:06.528 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=1004600, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=1004480, agent_async_task_token_map={"1004596":"2e0a0539-3934-4b25-939a-fb56b2e145af"}, task_operation=retry, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, primary_zone=, cluster_id=1000002, sub_task_instance_id=1004600, cluster_version=4.3.1.0, cluster_name=OB, target_server_status=RUNNING, startup_option_string=, ob_version=4.3.1.0, service_name=OB:1721059595, target_zone_status=RUNNING, ob_cluster_id=1721059595, zone1.idc=cs, root_sys_password=******, service_type=OB_CLUSTER, rpm_name=oceanbase-4.3.1.0-100010022024052916.el7.x86_64.rpm, connection_mode=direct, target_cluster_status=RUNNING, latest_execution_start_time=2024-07-16T10:34:06.513+08:00, ob_data_disk_path=/data/1}, listMap={root_servers=[1000001], ob_connect_address_list=[172.21.34.5:2881], server_ids=[1000002], dep_rpm_names=[], host_ids=[1000001], zone1.host_ids=[1000001], zone_names=[zone1]}}, executor=172.17.0.1 2024-07-16 10:34:06.531 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2024-07-16 10:34:06.550 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:34:06.576 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2024-07-16 10:34:06.578 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:34:07.094 ERROR 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:34:07.098 ERROR 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:34:07.102 WARN 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:34:07.106 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:34:12.109 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:34:12.113 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:34:12.118 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] ob connector is evicted from holder's cache. cause=EXPLICIT, key=ObConnectorKey(connectionMode=direct, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=172.21.34.5, port=2881, database=oceanbase) 2024-07-16 10:34:12.121 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alibaba.druid.pool.DruidDataSource : {dataSource-28} closing ... 2024-07-16 10:34:12.131 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alibaba.druid.pool.DruidDataSource : {dataSource-28} closed 2024-07-16 10:34:12.149 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:34:12.181 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:34:12.190 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:34:12.194 WARN 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:34:12.197 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:34:17.200 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:34:17.218 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:34:17.247 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:34:17.252 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:34:17.255 WARN 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:34:17.258 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:34:22.263 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:34:22.279 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:34:22.307 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:34:22.310 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:34:22.312 WARN 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:34:22.315 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:34:27.318 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:34:27.334 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:34:27.362 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:34:27.367 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:34:27.370 WARN 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:34:27.373 INFO 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 1004600, current state: RUNNING, new state: FAILED 2024-07-16 10:34:27.376 WARN 50 --- [pool-manual-subtask-executor6,23edc2ef01174bb9,919ef558143c] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=1004600, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=20, startTime=2024-07-16T10:34:06.513+08:00, endTime=2024-07-16T10:34:27.375+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:237) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.e ngine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.e xecuteUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] ... 14 common frames omitted ############{RETRY}{2024-07-16T10:38:35.470+08:00}############ 2024-07-16 10:38:35.479 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=1004600, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=1004480, agent_async_task_token_map={"1004596":"2e0a0539-3934-4b25-939a-fb56b2e145af"}, task_operation=retry, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, primary_zone=, cluster_id=1000002, sub_task_instance_id=1004600, cluster_version=4.3.1.0, cluster_name=OB, target_server_status=RUNNING, startup_option_string=, ob_version=4.3.1.0, service_name=OB:1721059595, target_zone_status=RUNNING, ob_cluster_id=1721059595, zone1.idc=cs, root_sys_password=******, service_type=OB_CLUSTER, rpm_name=oceanbase-4.3.1.0-100010022024052916.el7.x86_64.rpm, connection_mode=direct, target_cluster_status=RUNNING, latest_execution_start_time=2024-07-16T10:38:35.464+08:00, ob_data_disk_path=/data/1}, listMap={root_servers=[1000001], ob_connect_address_list=[172.21.34.5:2881], server_ids=[1000002], dep_rpm_names=[], host_ids=[1000001], zone1.host_ids=[1000001], zone_names=[zone1]}}, executor=172.17.0.1 2024-07-16 10:38:35.483 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2024-07-16 10:38:35.509 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:38:35.549 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2024-07-16 10:38:35.553 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:38:36.260 ERROR 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:38:36.265 ERROR 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:38:36.269 WARN 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:38:36.273 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:38:41.276 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:38:41.296 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:38:41.336 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:38:41.340 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:38:41.343 WARN 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:38:41.345 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:38:46.348 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:38:46.361 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:38:46.392 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:38:46.396 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:38:46.400 WARN 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:38:46.403 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:38:51.406 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:38:51.423 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:38:51.456 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:38:51.460 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:38:51.463 WARN 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:38:51.466 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:38:56.470 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:38:56.491 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:38:56.521 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:38:56.524 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:38:56.527 WARN 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:38:56.530 INFO 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 1004600, current state: RUNNING, new state: FAILED 2024-07-16 10:38:56.533 WARN 50 --- [pool-manual-subtask-executor7,23edc2ef01174bb9,11d96771ddee] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=1004600, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=20, startTime=2024-07-16T10:38:35.464+08:00, endTime=2024-07-16T10:38:56.532+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:237) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.e ngine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.e xecuteUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] ... 14 common frames omitted ############{RETRY}{2024-07-16T10:42:11.293+08:00}############ 2024-07-16 10:42:11.300 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=1004600, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=1004480, agent_async_task_token_map={"1004596":"2e0a0539-3934-4b25-939a-fb56b2e145af"}, task_operation=retry, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, primary_zone=, cluster_id=1000002, sub_task_instance_id=1004600, cluster_version=4.3.1.0, cluster_name=OB, target_server_status=RUNNING, startup_option_string=, ob_version=4.3.1.0, service_name=OB:1721059595, target_zone_status=RUNNING, ob_cluster_id=1721059595, zone1.idc=cs, root_sys_password=******, service_type=OB_CLUSTER, rpm_name=oceanbase-4.3.1.0-100010022024052916.el7.x86_64.rpm, connection_mode=direct, target_cluster_status=RUNNING, latest_execution_start_time=2024-07-16T10:42:11.286+08:00, ob_data_disk_path=/data/1}, listMap={root_servers=[1000001], ob_connect_address_list=[172.21.34.5:2881], server_ids=[1000002], dep_rpm_names=[], host_ids=[1000001], zone1.host_ids=[1000001], zone_names=[zone1]}}, executor=172.17.0.1 2024-07-16 10:42:11.304 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2024-07-16 10:42:11.323 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:42:11.355 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2024-07-16 10:42:11.358 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:42:12.124 ERROR 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:42:12.128 ERROR 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:42:12.132 WARN 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:42:12.135 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:42:17.139 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:42:17.154 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:42:17.178 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:42:17.184 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:42:17.187 WARN 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:42:17.189 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:42:22.191 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:42:22.212 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:42:22.243 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:42:22.246 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:42:22.249 WARN 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:42:22.251 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:42:27.254 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:42:27.272 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:42:27.308 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:42:27.313 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:42:27.316 WARN 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:42:27.319 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:42:32.323 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:42:32.341 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:42:32.378 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:42:32.382 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:42:32.385 WARN 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:42:32.388 INFO 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 1004600, current state: RUNNING, new state: FAILED 2024-07-16 10:42:32.392 WARN 50 --- [pool-manual-subtask-executor8,23edc2ef01174bb9,5e6c51cc4a95] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=1004600, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=20, startTime=2024-07-16T10:42:11.287+08:00, endTime=2024-07-16T10:42:32.391+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:237) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.e ngine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.e xecuteUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] ... 14 common frames omitted ############{RETRY}{2024-07-16T10:44:41.827+08:00}############ 2024-07-16 10:44:41.836 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=1004600, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=1004480, agent_async_task_token_map={"1004596":"2e0a0539-3934-4b25-939a-fb56b2e145af"}, task_operation=retry, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, primary_zone=, cluster_id=1000002, sub_task_instance_id=1004600, cluster_version=4.3.1.0, cluster_name=OB, target_server_status=RUNNING, startup_option_string=, ob_version=4.3.1.0, service_name=OB:1721059595, target_zone_status=RUNNING, ob_cluster_id=1721059595, zone1.idc=cs, root_sys_password=******, service_type=OB_CLUSTER, rpm_name=oceanbase-4.3.1.0-100010022024052916.el7.x86_64.rpm, connection_mode=direct, target_cluster_status=RUNNING, latest_execution_start_time=2024-07-16T10:44:41.820+08:00, ob_data_disk_path=/data/1}, listMap={root_servers=[1000001], ob_connect_address_list=[172.21.34.5:2881], server_ids=[1000002], dep_rpm_names=[], host_ids=[1000001], zone1.host_ids=[1000001], zone_names=[zone1]}}, executor=172.17.0.1 2024-07-16 10:44:41.839 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2024-07-16 10:44:41.858 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:44:41.895 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2024-07-16 10:44:41.898 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:44:42.892 ERROR 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:44:42.895 ERROR 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:44:42.898 WARN 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:44:42.900 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:44:47.903 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:44:47.920 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:44:47.946 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:44:47.949 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:44:47.952 WARN 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:44:47.955 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:44:52.958 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:44:52.977 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:44:53.006 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:44:53.011 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:44:53.014 WARN 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:44:53.016 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:44:58.019 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:44:58.038 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:44:58.068 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:44:58.072 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:44:58.075 WARN 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:44:58.078 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:45:03.081 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:45:03.097 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:45:03.127 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:45:03.131 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:45:03.134 WARN 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:45:03.137 INFO 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 1004600, current state: RUNNING, new state: FAILED 2024-07-16 10:45:03.143 WARN 50 --- [pool-manual-subtask-executor9,23edc2ef01174bb9,e8586ac6fb44] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=1004600, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=20, startTime=2024-07-16T10:44:41.820+08:00, endTime=2024-07-16T10:45:03.142+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:237) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.e ngine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.e xecuteUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] ... 14 common frames omitted ############{RETRY}{2024-07-16T10:51:49.347+08:00}############ 2024-07-16 10:51:49.359 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=1004600, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=1004480, agent_async_task_token_map={"1004596":"2e0a0539-3934-4b25-939a-fb56b2e145af"}, task_operation=retry, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, primary_zone=, cluster_id=1000002, sub_task_instance_id=1004600, cluster_version=4.3.1.0, cluster_name=OB, target_server_status=RUNNING, startup_option_string=, ob_version=4.3.1.0, service_name=OB:1721059595, target_zone_status=RUNNING, ob_cluster_id=1721059595, zone1.idc=cs, root_sys_password=******, service_type=OB_CLUSTER, rpm_name=oceanbase-4.3.1.0-100010022024052916.el7.x86_64.rpm, connection_mode=direct, target_cluster_status=RUNNING, latest_execution_start_time=2024-07-16T10:51:49.342+08:00, ob_data_disk_path=/data/1}, listMap={root_servers=[1000001], ob_connect_address_list=[172.21.34.5:2881], server_ids=[1000002], dep_rpm_names=[], host_ids=[1000001], zone1.host_ids=[1000001], zone_names=[zone1]}}, executor=172.17.0.1 2024-07-16 10:51:49.363 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2024-07-16 10:51:49.387 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:51:49.432 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2024-07-16 10:51:49.436 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:51:49.604 ERROR 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:51:49.608 ERROR 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:51:49.612 WARN 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:51:49.616 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:51:54.618 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:51:54.632 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:51:54.664 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:51:54.669 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:51:54.673 WARN 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:51:54.676 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:51:59.679 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:51:59.695 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:51:59.718 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:51:59.721 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:51:59.724 WARN 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:51:59.726 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:52:04.730 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:52:04.748 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:52:04.778 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:52:04.785 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:52:04.788 WARN 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:52:04.792 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:52:09.796 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:52:09.815 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:52:09.847 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:52:09.852 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:52:09.855 WARN 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:52:09.859 INFO 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 1004600, current state: RUNNING, new state: FAILED 2024-07-16 10:52:09.863 WARN 50 --- [pool-manual-subtask-executor10,23edc2ef01174bb9,b991fbee56fa] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=1004600, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=20, startTime=2024-07-16T10:51:49.342+08:00, endTime=2024-07-16T10:52:09.862+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:237) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task. engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry. executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] ... 14 common frames omitted ############{RETRY}{2024-07-16T10:56:23.333+08:00}############ 2024-07-16 10:56:23.340 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=1004600, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=1004480, agent_async_task_token_map={"1004596":"2e0a0539-3934-4b25-939a-fb56b2e145af"}, task_operation=retry, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, primary_zone=, cluster_id=1000002, sub_task_instance_id=1004600, cluster_version=4.3.1.0, cluster_name=OB, target_server_status=RUNNING, startup_option_string=, ob_version=4.3.1.0, service_name=OB:1721059595, target_zone_status=RUNNING, ob_cluster_id=1721059595, zone1.idc=cs, root_sys_password=******, service_type=OB_CLUSTER, rpm_name=oceanbase-4.3.1.0-100010022024052916.el7.x86_64.rpm, connection_mode=direct, target_cluster_status=RUNNING, latest_execution_start_time=2024-07-16T10:56:23.328+08:00, ob_data_disk_path=/data/1}, listMap={root_servers=[1000001], ob_connect_address_list=[172.21.34.5:2881], server_ids=[1000002], dep_rpm_names=[], host_ids=[1000001], zone1.host_ids=[1000001], zone_names=[zone1]}}, executor=172.17.0.1 2024-07-16 10:56:23.343 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2024-07-16 10:56:23.364 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:56:23.402 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2024-07-16 10:56:23.406 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:56:24.158 ERROR 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:56:24.164 ERROR 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 10:56:24.169 WARN 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:56:24.173 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:56:29.176 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:56:29.196 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:56:29.234 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:56:29.238 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:56:29.240 WARN 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:56:29.243 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:56:34.246 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:56:34.262 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:56:34.290 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:56:34.294 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:56:34.297 WARN 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:56:34.299 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:56:39.301 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:56:39.319 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:56:39.356 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:56:39.360 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:56:39.365 WARN 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:56:39.368 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 10:56:44.373 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 10:56:44.387 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 10:56:44.420 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 10:56:44.424 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 10:56:44.426 WARN 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 10:56:44.429 INFO 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 1004600, current state: RUNNING, new state: FAILED 2024-07-16 10:56:44.431 WARN 50 --- [pool-manual-subtask-executor11,23edc2ef01174bb9,267fdf80086d] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=1004600, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=20, startTime=2024-07-16T10:56:23.328+08:00, endTime=2024-07-16T10:56:44.431+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:237) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task. engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry. executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] ... 14 common frames omitted ############{RETRY}{2024-07-16T11:04:55.134+08:00}############ 2024-07-16 11:04:55.143 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=1004600, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=1004480, agent_async_task_token_map={"1004596":"2e0a0539-3934-4b25-939a-fb56b2e145af"}, task_operation=retry, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, primary_zone=, cluster_id=1000002, sub_task_instance_id=1004600, cluster_version=4.3.1.0, cluster_name=OB, target_server_status=RUNNING, startup_option_string=, ob_version=4.3.1.0, service_name=OB:1721059595, target_zone_status=RUNNING, ob_cluster_id=1721059595, zone1.idc=cs, root_sys_password=******, service_type=OB_CLUSTER, rpm_name=oceanbase-4.3.1.0-100010022024052916.el7.x86_64.rpm, connection_mode=direct, target_cluster_status=RUNNING, latest_execution_start_time=2024-07-16T11:04:55.129+08:00, ob_data_disk_path=/data/1}, listMap={root_servers=[1000001], ob_connect_address_list=[172.21.34.5:2881], server_ids=[1000002], dep_rpm_names=[], host_ids=[1000001], zone1.host_ids=[1000001], zone_names=[zone1]}}, executor=172.17.0.1 2024-07-16 11:04:55.146 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2024-07-16 11:04:55.165 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:04:55.206 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2024-07-16 11:04:55.210 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:04:55.366 ERROR 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 11:04:55.372 ERROR 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 11:04:55.376 WARN 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:04:55.379 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:05:00.383 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:05:00.399 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:05:00.425 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:05:00.428 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:05:00.430 WARN 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:05:00.432 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:05:05.434 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:05:05.447 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:05:05.480 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:05:05.484 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:05:05.486 WARN 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:05:05.488 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:05:10.490 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:05:10.509 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:05:10.543 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:05:10.547 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:05:10.551 WARN 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:05:10.554 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:05:15.559 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:05:15.581 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:05:15.614 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:05:15.619 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:05:15.622 WARN 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:05:15.626 INFO 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 1004600, current state: RUNNING, new state: FAILED 2024-07-16 11:05:15.629 WARN 50 --- [pool-manual-subtask-executor12,23edc2ef01174bb9,1ebd0cc36ad2] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=1004600, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=20, startTime=2024-07-16T11:04:55.129+08:00, endTime=2024-07-16T11:05:15.628+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:237) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task. engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry. executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] ... 14 common frames omitted ############{RETRY}{2024-07-16T11:11:27.573+08:00}############ 2024-07-16 11:11:27.580 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=1004600, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=1004480, agent_async_task_token_map={"1004596":"2e0a0539-3934-4b25-939a-fb56b2e145af"}, task_operation=retry, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, primary_zone=, cluster_id=1000002, sub_task_instance_id=1004600, cluster_version=4.3.1.0, cluster_name=OB, target_server_status=RUNNING, startup_option_string=, ob_version=4.3.1.0, service_name=OB:1721059595, target_zone_status=RUNNING, ob_cluster_id=1721059595, zone1.idc=cs, root_sys_password=******, service_type=OB_CLUSTER, rpm_name=oceanbase-4.3.1.0-100010022024052916.el7.x86_64.rpm, connection_mode=direct, target_cluster_status=RUNNING, latest_execution_start_time=2024-07-16T11:11:27.569+08:00, ob_data_disk_path=/data/1}, listMap={root_servers=[1000001], ob_connect_address_list=[172.21.34.5:2881], server_ids=[1000002], dep_rpm_names=[], host_ids=[1000001], zone1.host_ids=[1000001], zone_names=[zone1]}}, executor=172.17.0.1 2024-07-16 11:11:27.582 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2024-07-16 11:11:27.600 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:11:27.637 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2024-07-16 11:11:27.640 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:11:28.515 ERROR 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 11:11:28.519 ERROR 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 11:11:28.521 WARN 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:11:28.523 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:11:33.527 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:11:33.543 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:11:33.580 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:11:33.584 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:11:33.587 WARN 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:11:33.590 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:11:38.593 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:11:38.608 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:11:38.639 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:11:38.643 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:11:38.646 WARN 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:11:38.651 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:11:43.654 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:11:43.671 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:11:43.700 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:11:43.704 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:11:43.706 WARN 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:11:43.708 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:11:48.710 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:11:48.726 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:11:48.753 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:11:48.756 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:11:48.759 WARN 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:11:48.762 INFO 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 1004600, current state: RUNNING, new state: FAILED 2024-07-16 11:11:48.765 WARN 50 --- [pool-manual-subtask-executor13,23edc2ef01174bb9,540c142b6a84] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=1004600, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=20, startTime=2024-07-16T11:11:27.570+08:00, endTime=2024-07-16T11:11:48.764+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:237) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task. engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry. executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] ... 14 common frames omitted ############{RETRY}{2024-07-16T11:26:19.681+08:00}############ 2024-07-16 11:26:19.688 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.c.m.t.model.SubtaskInstanceEntity : Retry subtask, id=1004600, context=Context{parallelIdx=-1, stringMap={prohibit_rollback=false, ob_log_disk_path=/data/log1, task_instance_id=1004480, agent_async_task_token_map={"1004596":"2e0a0539-3934-4b25-939a-fb56b2e145af"}, task_operation=retry, cluster_type=PRIMARY, ob_install_path=/home/admin/oceanbase, link_obproxy=false, primary_zone=, cluster_id=1000002, sub_task_instance_id=1004600, cluster_version=4.3.1.0, cluster_name=OB, target_server_status=RUNNING, startup_option_string=, ob_version=4.3.1.0, service_name=OB:1721059595, target_zone_status=RUNNING, ob_cluster_id=1721059595, zone1.idc=cs, root_sys_password=******, service_type=OB_CLUSTER, rpm_name=oceanbase-4.3.1.0-100010022024052916.el7.x86_64.rpm, connection_mode=direct, target_cluster_status=RUNNING, latest_execution_start_time=2024-07-16T11:26:19.676+08:00, ob_data_disk_path=/data/1}, listMap={root_servers=[1000001], ob_connect_address_list=[172.21.34.5:2881], server_ids=[1000002], dep_rpm_names=[], host_ids=[1000001], zone1.host_ids=[1000001], zone_names=[zone1]}}, executor=172.17.0.1 2024-07-16 11:26:19.691 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.t.b.cluster.SyncClusterInfoTask : need to do nothing when rollback 2024-07-16 11:26:19.714 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:26:19.759 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=proxy, clusterName=OB, obClusterId=1721059595, tenantName=sys, username=ocp_monitor, address=localhost, port=2888, database=oceanbase) 2024-07-16 11:26:19.763 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] create new ob connector, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:26:20.327 ERROR 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.core.obsdk.connector.ObConnectors : [obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 11:26:20.330 ERROR 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.o.f.AbstractObOperatorFactory : [AbstractObOperatorFactory] create operator failed, error message:[obsdk] init ob connector failed, connectProperties=ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase), cause:java.sql.SQLNonTransientConnectionException: Could not connect to address=(host=localhost)(port=2888)(type=master) : Could not connect to localhost:2888 : unexpected end of stream, read 0 bytes from 4 (socket was closed by server) 2024-07-16 11:26:20.333 WARN 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:26:20.336 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:26:25.338 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:26:25.354 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:26:25.387 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:26:25.392 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:26:25.394 WARN 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:26:25.397 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:26:30.399 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:26:30.419 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:26:30.449 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:26:30.453 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:26:30.456 WARN 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:26:30.458 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:26:35.461 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:26:35.478 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:26:35.510 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:26:35.514 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:26:35.518 WARN 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:26:35.521 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] com.alipay.ocp.common.pattern.Retry : wait for 5 seconds 2024-07-16 11:26:40.524 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.c.o.connector.ObConnectorHolder : [obsdk] invalidate ob connector, clusterName=OB 2024-07-16 11:26:40.542 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.ob.cluster.ClusterSyncService : sync cluster start, clusterId=1000002, clusterName=OB, obClusterId=1721059595 2024-07-16 11:26:40.576 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=OB, tenantName=sys, dbUser=ocp_monitor 2024-07-16 11:26:40.579 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.s.o.o.f.AbstractObOperatorFactory : get connect state from obsdk context, state: false, connectProperties:ConnectProperties(connectionMode=proxy, address=localhost, port=2888, obsAddrList=null, username=ocp_monitor, tenantName=sys, clusterName=OB, obClusterId=1721059595, proxy=null, vpcId=1, compatibilityMode=MYSQL, database=oceanbase) 2024-07-16 11:26:40.582 WARN 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] com.alipay.ocp.common.pattern.Retry : failed to execute target runnable, errMsg:[OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor 2024-07-16 11:26:40.584 INFO 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.c.m.t.model.SubtaskInstanceEntity : Set state for subtask: 1004600, current state: RUNNING, new state: FAILED 2024-07-16 11:26:40.588 WARN 50 --- [pool-manual-subtask-executor14,23edc2ef01174bb9,6be57275d667] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceEntity{id=1004600, name=Sync cluster info, state=FAILED, operation=RETRY, className=com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask, seriesId=20, startTime=2024-07-16T11:26:19.677+08:00, endTime=2024-07-16T11:26:40.587+08:00} java.lang.RuntimeException: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:173) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.executeUntilSuccessWithLimit(Retry.java:188) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.run(SyncClusterInfoTask.java:44) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.metadb.task.model.SubtaskInstanceEntity.retry(SubtaskInstanceEntity.java:237) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.doExecute(JavaTaskRunner.java:30) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.JavaTaskRunner.run(JavaTaskRunner.java:20) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:118) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:190) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:107) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.core.task. engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$submitTask$3(ReadySubtaskWorker.java:122) ~[ocp-core-3.3.4-20221201.jar!/:3.3.4-20221201] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Caused by: com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=OB_CLUSTER_CONNECT_FAILED, args=OB,ocp_monitor at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.getObOperator(AbstractObOperatorFactory.java:161) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:121) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.oceanbase.obsdk.factory.AbstractObOperatorFactory.createObOperator(AbstractObOperatorFactory.java:125) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.zoneEntitiesCurrent(ClusterSyncService.java:530) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.internalSyncCluster(ClusterSyncService.java:241) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.operation.ob.cluster.ClusterSyncService.syncClusterInfo(ClusterSyncService.java:192) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.service.task.business.cluster.SyncClusterInfoTask.lambda$run$0(SyncClusterInfoTask.java:45) ~[ocp-service-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry.lambda$executeUntilSuccessWithLimit$0(Retry.java:189) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] at com.alipay.ocp.common.pattern.Retry. executeUntilSuccessWithLimit(Retry.java:169) ~[ocp-common-3.3.4-20221201.jar!/:3.3.4-20221201] ... 14 common frames omitted