2024-09-11 14:11:41.396 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.o.c.t.e.runner.JavaSubtaskRunner : Run subtask, id=2008969, context=Context{parallelIdx=0, stringMap={ob_log_disk_path=/redo/ob, task_instance_id=2006662, task_operation=execute, ob_run_path=/home/admin/oceanbase, service_version=4.3.1.0, ob_install_path=/home/admin/oceanbase, host.2000007.zone=zone1, ob_sql_port=2881, cluster_id=1000001, ob_disk_path_style=OBD, ob_svr_port=2882, sub_task_instance_name=Add observer, sub_task_instance_id=2008969, cluster_name=obcluster, target_server_status=RUNNING, mix_rpm_names={"zone3":"oceanbase-ce-4.3.1.0-100000032024051615.el7.x86_64.rpm","zone2":"oceanbase-ce-4.3.1.0-100000032024051615.el7.x86_64.rpm","zone1":"oceanbase-ce-4.3.1.0-100000032024051615.el7.x86_64.rpm"}, startup_option_string=max_syslog_file_count=4,enable_syslog_recycle=TRUE,enable_syslog_wf=FALSE,syslog_level=INFO,system_memory=16G,devname=eth0,memory_limit=240G, subtask_splitter=host_ids, service_name=obcluster:1723810551, target_zone_status=RUNNING, ob_cluster_id=1723810551, service_type=OB_CLUSTER, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.3.1.0-100000032024051615.el7.x86_64.rpm","oceanbase-ce-utils-4.3.1.0-100000032024051615.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.3.1.0-100000032024051615.el7.x86_64.rpm","oceanbase-ce-utils-4.3.1.0-100000032024051615.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.3.1.0-100000032024051615.el7.x86_64.rpm","oceanbase-ce-utils-4.3.1.0-100000032024051615.el7.x86_64.rpm"]}, ob_run_user=admin, target_cluster_status=RUNNING, latest_execution_start_time=2024-09-11T14:11:41.337+08:00, ob_data_disk_path=/data/ob, mix_obs_rpm={}}, listMap={root_server_ips=[10.21.8.26, 10.21.8.27, 10.21.8.28], server_ids=[1000009], exists_server_addrs=[10.21.8.26:2882, 10.21.8.27:2882, 10.21.8.28:2882], host_ids=[2000007], exists_running_server_addrs=[10.21.8.26:2882, 10.21.8.27:2882, 10.21.8.28:2882], zone_names=[zone1]}}, executor=10.21.8.50 2024-09-11 14:11:41.436 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.27, port=2881, database=oceanbase) 2024-09-11 14:11:41.437 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.28, port=2881, database=oceanbase) 2024-09-11 14:11:41.438 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.o.s.t.b.server.ObServerTaskHandler : begin to add server, zoneName=zone1, svrIp=10.21.8.23, svrPort=2882 2024-09-11 14:11:41.441 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.27, port=2881, database=oceanbase) 2024-09-11 14:11:41.443 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.28, port=2881, database=oceanbase) 2024-09-11 14:11:41.444 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-09-11 14:11:41.447 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT zone, svr_ip, svr_port, sql_port AS inner_port, with_rootserver, UPPER(`status`) AS `status`, build_version, UNIX_TIMESTAMP(NVL(`stop_time`, 0))*1000000 AS stop_time, UNIX_TIMESTAMP(NVL(`start_service_time`, 0))*1000000 AS start_service_time, UNIX_TIMESTAMP(NVL(`block_migrate_in_time`, 0))*1000000 AS block_migrate_in_time FROM DBA_OB_SERVERS WHERE svr_ip = ? AND svr_port = ?, args: [10.21.8.23, 2882] 2024-09-11 14:11:41.448 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.27, port=2881, database=oceanbase) 2024-09-11 14:11:41.449 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.28, port=2881, database=oceanbase) 2024-09-11 14:11:41.451 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-09-11 14:11:41.452 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [600000000] 2024-09-11 14:11:41.454 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: alter system add server ? zone ?, args: [10.21.8.23:2882, zone1] 2024-09-11 14:21:41.408 WARN 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] update failed, sql:[alter system add server ? zone ?], error message:[PreparedStatementCallback; SQL [alter system add server ? zone ?]; (conn=3221585428) Timeout; nested exception is java.sql.SQLTransientConnectionException: (conn=3221585428) Timeout] 2024-09-11 14:21:41.410 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.ocp.obsdk.connector.ConnectTemplate : Last Trace Info:[YB420A15081A-000621C87DEFEAEE-0-0] 2024-09-11 14:21:41.411 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] slow query, durationMillis=599957, sql=alter system add server ? zone ? 2024-09-11 14:21:41.412 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-09-11 14:21:41.414 ERROR 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,21451a6148f5eef6] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Timeout java.sql.SQLException: Timeout at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.readErrorPacket(AbstractQueryProtocol.java:2347) at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.readPacket(AbstractQueryProtocol.java:2210) at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.getResult(AbstractQueryProtocol.java:2098) at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.executeQuery(AbstractQueryProtocol.java:399) at com.oceanbase.jdbc.JDBC4PreparedStatement.executeInternal(JDBC4PreparedStatement.java:247) at com.oceanbase.jdbc.JDBC4PreparedStatement.execute(JDBC4PreparedStatement.java:170) at com.oceanbase.jdbc.JDBC4PreparedStatement.executeUpdate(JDBC4PreparedStatement.java:204) at com.alibaba.druid.pool.DruidPooledPreparedStatement.executeUpdate(DruidPooledPreparedStatement.java:255) at org.springframework.jdbc.core.JdbcTemplate.lambda$update$2(JdbcTemplate.java:967) at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:650) at org.springframework.jdbc.core.JdbcTemplate.update(JdbcTemplate.java:962) at org.springframework.jdbc.core.JdbcTemplate.update(JdbcTemplate.java:1017) at org.springframework.jdbc.core.JdbcTemplate.update(JdbcTemplate.java:1027) at com.oceanbase.ocp.obsdk.connector.ConnectTemplate.updateInner(ConnectTemplate.java:293) at com.oceanbase.ocp.obsdk.connector.ConnectTemplate.update(ConnectTemplate.java:282) at com.oceanbase.ocp.obsdk.operator.cluster.MysqlClusterOperator.addServer(MysqlClusterOperator.java:613) at com.oceanbase.ocp.service.task.business.server.ObServerTaskHandler.addObServer(ObServerTaskHandler.java:196) at com.oceanbase.ocp.service.task.business.server.AddObServerTask.run(AddObServerTask.java:53) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.execute(JavaSubtaskRunner.java:64) at com.oceanbase.ocp.core.task.engine.runner.JavaSubta skRunner.doRun(JavaSubtaskRunner.java:32) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:206) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:200) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:137) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 2008969, operation:EXECUTE, state: FAILED 2024-09-11 14:46:40.724 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=2008969, context=Context{parallelIdx=0, stringMap={ob_log_disk_path=/redo/ob, task_instance_id=2006662, task_operation=retry, ob_run_path=/home/admin/oceanbase, service_version=4.3.1.0, ob_install_path=/home/admin/oceanbase, host.2000007.zone=zone1, ob_sql_port=2881, cluster_id=1000001, ob_disk_path_style=OBD, ob_svr_port=2882, sub_task_instance_name=Add observer, sub_task_instance_id=2008969, cluster_name=obcluster, target_server_status=RUNNING, mix_rpm_names={"zone3":"oceanbase-ce-4.3.1.0-100000032024051615.el7.x86_64.rpm","zone2":"oceanbase-ce-4.3.1.0-100000032024051615.el7.x86_64.rpm","zone1":"oceanbase-ce-4.3.1.0-100000032024051615.el7.x86_64.rpm"}, startup_option_string=max_syslog_file_count=4,enable_syslog_recycle=TRUE,enable_syslog_wf=FALSE,syslog_level=INFO,system_memory=16G,devname=eth0,memory_limit=240G, subtask_splitter=host_ids, service_name=obcluster:1723810551, target_zone_status=RUNNING, ob_cluster_id=1723810551, service_type=OB_CLUSTER, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.3.1.0-100000032024051615.el7.x86_64.rpm","oceanbase-ce-utils-4.3.1.0-100000032024051615.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.3.1.0-100000032024051615.el7.x86_64.rpm","oceanbase-ce-utils-4.3.1.0-100000032024051615.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.3.1.0-100000032024051615.el7.x86_64.rpm","oceanbase-ce-utils-4.3.1.0-100000032024051615.el7.x86_64.rpm"]}, ob_run_user=admin, target_cluster_status=RUNNING, latest_execution_start_time=2024-09-11T14:46:40.655+08:00, ob_data_disk_path=/data/ob, mix_obs_rpm={}}, listMap={root_server_ips=[10.21.8.26, 10.21.8.27, 10.21.8.28], server_ids=[1000009], exists_server_addrs=[10.21.8.26:2882, 10.21.8.27:2882, 10.21.8.28:2882], host_ids=[2000007], exists_running_server_addrs=[10.21.8.26:2882, 10.21.8.27:2882, 10.21.8.28:2882], zone_names=[zone1]}}, executor=10.21.8.50 2024-09-11 14:46:40.748 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.27, port=2881, database=oceanbase) 2024-09-11 14:46:40.749 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.28, port=2881, database=oceanbase) 2024-09-11 14:46:40.750 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.o.s.t.b.server.ObServerTaskHandler : begin to add server, zoneName=zone1, svrIp=10.21.8.23, svrPort=2882 2024-09-11 14:46:40.752 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.27, port=2881, database=oceanbase) 2024-09-11 14:46:40.753 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.28, port=2881, database=oceanbase) 2024-09-11 14:46:40.754 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-09-11 14:46:40.757 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT zone, svr_ip, svr_port, sql_port AS inner_port, with_rootserver, UPPER(`status`) AS `status`, build_version, UNIX_TIMESTAMP(NVL(`stop_time`, 0))*1000000 AS stop_time, UNIX_TIMESTAMP(NVL(`start_service_time`, 0))*1000000 AS start_service_time, UNIX_TIMESTAMP(NVL(`block_migrate_in_time`, 0))*1000000 AS block_migrate_in_time FROM DBA_OB_SERVERS WHERE svr_ip = ? AND svr_port = ?, args: [10.21.8.23, 2882] 2024-09-11 14:46:40.758 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.27, port=2881, database=oceanbase) 2024-09-11 14:46:40.759 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.28, port=2881, database=oceanbase) 2024-09-11 14:46:40.760 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-09-11 14:46:40.761 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [600000000] 2024-09-11 14:46:40.763 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: alter system add server ? zone ?, args: [10.21.8.23:2882, zone1] 2024-09-11 14:56:40.760 WARN 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] update failed, sql:[alter system add server ? zone ?], error message:[PreparedStatementCallback; SQL [alter system add server ? zone ?]; (conn=3221585428) Timeout; nested exception is java.sql.SQLTransientConnectionException: (conn=3221585428) Timeout] 2024-09-11 14:56:40.762 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.ocp.obsdk.connector.ConnectTemplate : Last Trace Info:[YB420A15081A-000621C87DEFF82E-0-0] 2024-09-11 14:56:40.763 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] slow query, durationMillis=600000, sql=alter system add server ? zone ? 2024-09-11 14:56:40.764 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-09-11 14:56:40.765 ERROR 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,bd495364cd92002a] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Timeout java.sql.SQLException: Timeout at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.readErrorPacket(AbstractQueryProtocol.java:2347) at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.readPacket(AbstractQueryProtocol.java:2210) at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.getResult(AbstractQueryProtocol.java:2098) at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.executeQuery(AbstractQueryProtocol.java:399) at com.oceanbase.jdbc.JDBC4PreparedStatement.executeInternal(JDBC4PreparedStatement.java:247) at com.oceanbase.jdbc.JDBC4PreparedStatement.execute(JDBC4PreparedStatement.java:170) at com.oceanbase.jdbc.JDBC4PreparedStatement.executeUpdate(JDBC4PreparedStatement.java:204) at com.alibaba.druid.pool.DruidPooledPreparedStatement.executeUpdate(DruidPooledPreparedStatement.java:255) at org.springframework.jdbc.core.JdbcTemplate.lambda$update$2(JdbcTemplate.java:967) at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:650) at org.springframework.jdbc.core.JdbcTemplate.update(JdbcTemplate.java:962) at org.springframework.jdbc.core.JdbcTemplate.update(JdbcTemplate.java:1017) at org.springframework.jdbc.core.JdbcTemplate.update(JdbcTemplate.java:1027) at com.oceanbase.ocp.obsdk.connector.ConnectTemplate.updateInner(ConnectTemplate.java:293) at com.oceanbase.ocp.obsdk.connector.ConnectTemplate.update(ConnectTemplate.java:282) at com.oceanbase.ocp.obsdk.operator.cluster.MysqlClusterOperator.addServer(MysqlClusterOperator.java:613) at com.oceanbase.ocp.service.task.business.server.ObServerTaskHandler.addObServer(ObServerTaskHandler.java:196) at com.oceanbase.ocp.service.task.business.server.AddObServerTask.run(AddObServerTask.java:53) at com.oceanbase.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.retry(JavaSubtaskRu nner.java:76) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:35) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:206) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:200) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:137) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 2008969, operation:RETRY, state: FAILED 2024-09-11 15:00:54.553 WARN 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.c.t.e.runner.JavaSubtaskRunner : Rollback subtask, id=2008969, context=Context{parallelIdx=0, stringMap={ob_log_disk_path=/redo/ob, task_instance_id=2006662, task_operation=rollback, ob_run_path=/home/admin/oceanbase, service_version=4.3.1.0, ob_install_path=/home/admin/oceanbase, host.2000007.zone=zone1, ob_sql_port=2881, cluster_id=1000001, ob_disk_path_style=OBD, ob_svr_port=2882, sub_task_instance_name=Add observer, sub_task_instance_id=2008969, cluster_name=obcluster, target_server_status=RUNNING, mix_rpm_names={"zone3":"oceanbase-ce-4.3.1.0-100000032024051615.el7.x86_64.rpm","zone2":"oceanbase-ce-4.3.1.0-100000032024051615.el7.x86_64.rpm","zone1":"oceanbase-ce-4.3.1.0-100000032024051615.el7.x86_64.rpm"}, startup_option_string=max_syslog_file_count=4,enable_syslog_recycle=TRUE,enable_syslog_wf=FALSE,syslog_level=INFO,system_memory=16G,devname=eth0,memory_limit=240G, subtask_splitter=host_ids, service_name=obcluster:1723810551, target_zone_status=RUNNING, ob_cluster_id=1723810551, service_type=OB_CLUSTER, dep_rpm_names={"zone3":["oceanbase-ce-libs-4.3.1.0-100000032024051615.el7.x86_64.rpm","oceanbase-ce-utils-4.3.1.0-100000032024051615.el7.x86_64.rpm"],"zone2":["oceanbase-ce-libs-4.3.1.0-100000032024051615.el7.x86_64.rpm","oceanbase-ce-utils-4.3.1.0-100000032024051615.el7.x86_64.rpm"],"zone1":["oceanbase-ce-libs-4.3.1.0-100000032024051615.el7.x86_64.rpm","oceanbase-ce-utils-4.3.1.0-100000032024051615.el7.x86_64.rpm"]}, ob_run_user=admin, target_cluster_status=RUNNING, latest_execution_start_time=2024-09-11T15:00:54.490+08:00, ob_data_disk_path=/data/ob, mix_obs_rpm={}}, listMap={root_server_ips=[10.21.8.26, 10.21.8.27, 10.21.8.28], server_ids=[1000009], exists_server_addrs=[10.21.8.26:2882, 10.21.8.27:2882, 10.21.8.28:2882], host_ids=[2000007], exists_running_server_addrs=[10.21.8.26:2882, 10.21.8.27:2882, 10.21.8.28:2882], zone_names=[zone1]}}, operation=ROLLBACK, executor=10.21.8.50 2024-09-11 15:00:54.555 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.s.t.b.server.AddObServerTask : begin to roll back 2024-09-11 15:00:54.575 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.27, port=2881, database=oceanbase) 2024-09-11 15:00:54.576 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.28, port=2881, database=oceanbase) 2024-09-11 15:00:54.577 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.s.t.b.server.ObServerTaskHandler : begin to delete server, svrIp=10.21.8.23, svrPort=2882 2024-09-11 15:00:54.578 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.27, port=2881, database=oceanbase) 2024-09-11 15:00:54.580 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=root, address=10.21.8.28, port=2881, database=oceanbase) 2024-09-11 15:00:54.581 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-09-11 15:00:54.584 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT zone, svr_ip, svr_port, sql_port AS inner_port, with_rootserver, UPPER(`status`) AS `status`, build_version, UNIX_TIMESTAMP(NVL(`stop_time`, 0))*1000000 AS stop_time, UNIX_TIMESTAMP(NVL(`start_service_time`, 0))*1000000 AS start_service_time, UNIX_TIMESTAMP(NVL(`block_migrate_in_time`, 0))*1000000 AS block_migrate_in_time FROM DBA_OB_SERVERS WHERE svr_ip = ? AND svr_port = ?, args: [10.21.8.23, 2882] 2024-09-11 15:00:54.585 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.s.t.b.server.ObServerTaskHandler : target observer is not in cluster, no need to execute delete server command 2024-09-11 15:00:54.586 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.o.i.helper.OcpCacheHelperImpl : Ob cluster resource has been changed, cluster 1000001, tenant null, notify ocp cache 2024-09-11 15:00:54.587 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.s.t.b.server.AddObServerTask : Begin to wait for observer delete success 2024-09-11 15:00:54.599 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=ocp_monitor, address=10.21.8.27, port=2881, database=oceanbase) 2024-09-11 15:00:54.600 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=ocp_monitor, address=10.21.8.28, port=2881, database=oceanbase) 2024-09-11 15:00:54.613 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.s.t.b.server.ObServerTaskHandler : start to check server active 2024-09-11 15:00:54.614 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=ocp_monitor, address=10.21.8.27, port=2881, database=oceanbase) 2024-09-11 15:00:54.615 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=obcluster, obClusterId=1723810551, tenantName=sys, username=ocp_monitor, address=10.21.8.28, port=2881, database=oceanbase) 2024-09-11 15:00:54.617 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2024-09-11 15:00:54.620 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT zone, svr_ip, svr_port, sql_port AS inner_port, with_rootserver, UPPER(`status`) AS `status`, build_version, UNIX_TIMESTAMP(NVL(`stop_time`, 0))*1000000 AS stop_time, UNIX_TIMESTAMP(NVL(`start_service_time`, 0))*1000000 AS start_service_time, UNIX_TIMESTAMP(NVL(`block_migrate_in_time`, 0))*1000000 AS block_migrate_in_time FROM DBA_OB_SERVERS WHERE svr_ip = ? AND svr_port = ?, args: [10.21.8.23, 2882] 2024-09-11 15:00:54.622 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.s.t.b.server.ObServerTaskHandler : server 10.21.8.23, obServer=null 2024-09-11 15:00:54.623 INFO 26467 --- [manual-subtask-executor15,a4cfea886a10a90f,319465b5a22f0dc6] c.o.o.s.t.b.server.AddObServerTask : Wait for observer delete success Set state for subtask: 2008969, operation:ROLLBACK, state: PENDING