【 使用环境 】生产环境
【 OB or 其他组件 】ob and ocp e
【 使用版本 】4.1.0
【问题描述】
OceanBase启动报错 failed to connect meta db ocp-express start failed observer need bootstarp
【复现路径】问题出现前后相关操作
【问题现象及影响】
$ obd cluster start myoceanbase -c ocp-express
Get local repositories ok
Search plugins ok
Open ssh connection ok
Load cluster param plugin ok
Check before start ocp-express ok
Start ocp-express ok
[ERROR] failed to start 192.168.112.39 ocp-express
[ERROR] ocp-express start failed
【附件】
ocp-express启动报错日志如下
[2023-05-24T15:06:51.330] END dataSource DEFAULT_DATA sqls:ocp_metric_expr_config
java.sql.SQLException: execute sql task failed. task: ocp_metric_expr_config, sql:INSERT INTO ocp_metric_expr_config
(metric
,expr
) VALUES (‘active_memstore_percent’,‘100 * sum(ob_sysstat{stat_id=“130000”,@LABELS}) by (@GBLABELS) / sum(ob_sysstat{stat_id=“130002”,@LABELS}) by (@GBLABELS)’) ON DUPLICATE KEY UPDATE expr
=‘100 * sum(ob_sysstat{stat_id=“130000”,@LABELS}) by (@GBLABELS) / sum(ob_sysstat{stat_id=“130002”,@LABELS}) by (@GBLABELS)’
at com.oceanbase.ocp.bootstrap.db.DbInitializer.executeSqlTask(DbInitializer.java:216)
at com.oceanbase.ocp.bootstrap.db.DbInitializer.writeDefaultData(DbInitializer.java:254)
at com.oceanbase.ocp.bootstrap.db.DbInitializer.install(DbInitializer.java:122)
at com.oceanbase.ocp.bootstrap.db.DbInitializer.initialize(DbInitializer.java:101)
at com.oceanbase.ocp.bootstrap.spring.DBInitInterceptor.afterDataSourceCreation(DBInitInterceptor.java:70)
at com.oceanbase.ocp.bootstrap.spring.DataSourceInterceptor.lambda$afterDataSourceCreationHook$0(DataSourceInterceptor.java:43)
at java.util.ArrayList.forEach(ArrayList.java:1259)
at com.oceanbase.ocp.bootstrap.spring.DataSourceInterceptor.afterDataSourceCreationHook(DataSourceInterceptor.java:41)
at com.oceanbase.ocp.bootstrap.spring.DataSourceInterceptor.postProcessAfterInitialization(DataSourceInterceptor.java:34)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyBeanPostProcessorsAfterInitialization(AbstractAutowireCapableBeanFactory.java:455)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1808)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:620)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:542)
at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:335)
at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234)
at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:333)
at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:208)
at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:955)
at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:918)
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:583)
at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:745)
at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:420)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:307)
at org.springframework.boot.builder.SpringApplicationBuilder.run(SpringApplicationBuilder.java:164)
at org.springframework.cloud.bootstrap.BootstrapApplicationListener.bootstrapServiceContext(BootstrapApplicationListener.java:195)
at org.springframework.cloud.bootstrap.BootstrapApplicationListener.onApplicationEvent(BootstrapApplicationListener.java:114)
at org.springframework.cloud.bootstrap.BootstrapApplicationListener.onApplicationEvent(BootstrapApplicationListener.java:77)
at org.springframework.context.event.SimpleApplicationEventMulticaster.doInvokeListener(SimpleApplicationEventMulticaster.java:176)
at org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:169)
at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:143)
at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:131)
at org.springframework.boot.context.event.EventPublishingRunListener.environmentPrepared(EventPublishingRunListener.java:85)
at org.springframework.boot.SpringApplicationRunListeners.lambda$environmentPrepared$2(SpringApplicationRunListeners.java:66)
at java.util.ArrayList.forEach(ArrayList.java:1259)
at org.springframework.boot.SpringApplicationRunListeners.doWithListeners(SpringApplicationRunListeners.java:120)
at org.springframework.boot.SpringApplicationRunListeners.doWithListeners(SpringApplicationRunListeners.java:114)
at org.springframework.boot.SpringApplicationRunListeners.environmentPrepared(SpringApplicationRunListeners.java:65)
at org.springframework.boot.SpringApplication.prepareEnvironment(SpringApplication.java:343)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:301)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1317)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1306)
at com.oceanbase.ocp.OcpExpressApplication.main(OcpExpressApplication.java:18)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:49)
at org.springframework.boot.loader.Launcher.launch(Launcher.java:107)
at org.springframework.boot.loader.Launcher.launch(Launcher.java:58)
at org.springframework.boot.loader.PropertiesLauncher.main(PropertiesLauncher.java:467)
Caused by: java.sql.SQLNonTransientConnectionException: (conn=397) Connection timed out
at com.oceanbase.jdbc.internal.util.exceptions.ExceptionFactory.createException(ExceptionFactory.java:120)
at com.oceanbase.jdbc.internal.util.exceptions.ExceptionFactory.create(ExceptionFactory.java:200)
at com.oceanbase.jdbc.OceanBaseStatement.executeBatchExceptionEpilogue(OceanBaseStatement.java:370)
at com.oceanbase.jdbc.OceanBaseStatement.executeBatch(OceanBaseStatement.java:1487)
at com.oceanbase.ocp.bootstrap.util.SQLUtils.executeBatch(SQLUtils.java:99)
at com.oceanbase.ocp.bootstrap.db.DbInitializer.lambda$new$0(DbInitializer.java:93)
at com.oceanbase.ocp.bootstrap.db.DbInitializer.executeSqlTask(DbInitializer.java:203)
… 49 more
Caused by: com.oceanbase.jdbc.internal.util.exceptions.OceanBaseSqlException: Connection timed out
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.exceptionWithQuery(AbstractQueryProtocol.java:188)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.executeBatchAggregateSemiColon(AbstractQueryProtocol.java:1084)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.executeBatchStmt(AbstractQueryProtocol.java:943)
at com.oceanbase.jdbc.OceanBaseStatement.internalBatchExecution(OceanBaseStatement.java:1544)
at com.oceanbase.jdbc.OceanBaseStatement.executeBatch(OceanBaseStatement.java:1484)
… 52 more
Caused by: java.sql.SQLNonTransientConnectionException: Read timed out
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.handleIoException(AbstractQueryProtocol.java:2361)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.readPacket(AbstractQueryProtocol.java:1839)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.getResult(AbstractQueryProtocol.java:1748)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.executeBatchAggregateSemiColon(AbstractQueryProtocol.java:1080)
… 55 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at java.io.FilterInputStream.read(FilterInputStream.java:133)
at com.oceanbase.jdbc.internal.io.input.ReadAheadBufferedStream.fillBuffer(ReadAheadBufferedStream.java:148)
at com.oceanbase.jdbc.internal.io.input.ReadAheadBufferedStream.read(ReadAheadBufferedStream.java:121)
at com.oceanbase.jdbc.internal.io.input.StandardPacketInputStream.getPacketArray(StandardPacketInputStream.java:247)
at com.oceanbase.jdbc.internal.io.input.StandardPacketInputStream.getPacket(StandardPacketInputStream.java:217)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.readPacket(AbstractQueryProtocol.java:1837)
… 57 more
[2023-05-24T15:06:51.333] END dataSource INSTALL -
java.lang.IllegalStateException: java.sql.SQLException: execute sql task failed. task: ocp_metric_expr_config, sql:INSERT INTO ocp_metric_expr_config
(metric
,expr
) VALUES (‘active_memstore_percent’,‘100 * sum(ob_sysstat{stat_id=“130000”,@LABELS}) by (@GBLABELS) / sum(ob_sysstat{stat_id=“130002”,@LABELS}) by (@GBLABELS)’) ON DUPLICATE KEY UPDATE expr
=‘100 * sum(ob_sysstat{stat_id=“130000”,@LABELS}) by (@GBLABELS) / sum(ob_sysstat{stat_id=“130002”,@LABELS}) by (@GBLABELS)’
at com.oceanbase.ocp.bootstrap.db.DbInitializer.executeSqlTask(DbInitializer.java:218)
at com.oceanbase.ocp.bootstrap.db.DbInitializer.writeDefaultData(DbInitializer.java:254)
at com.oceanbase.ocp.bootstrap.db.DbInitializer.install(DbInitializer.java:122)
at com.oceanbase.ocp.bootstrap.db.DbInitializer.initialize(DbInitializer.java:101)
at com.oceanbase.ocp.bootstrap.spring.DBInitInterceptor.afterDataSourceCreation(DBInitInterceptor.java:70)
at com.oceanbase.ocp.bootstrap.spring.DataSourceInterceptor.lambda$afterDataSourceCreationHook$0(DataSourceInterceptor.java:43)
at java.util.ArrayList.forEach(ArrayList.java:1259)
at com.oceanbase.ocp.bootstrap.spring.DataSourceInterceptor.afterDataSourceCreationHook(DataSourceInterceptor.java:41)
at com.oceanbase.ocp.bootstrap.spring.DataSourceInterceptor.postProcessAfterInitialization(DataSourceInterceptor.java:34)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.applyBeanPostProcessorsAfterInitialization(AbstractAutowireCapableBeanFactory.java:455)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1808)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:620)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:542)
at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:335)
at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234)
at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:333)
at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:208)
at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:955)
at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:918)
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:583)
at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:745)
at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:420)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:307)
at org.springframework.boot.builder.SpringApplicationBuilder.run(SpringApplicationBuilder.java:164)
at org.springframework.cloud.bootstrap.BootstrapApplicationListener.bootstrapServiceContext(BootstrapApplicationListener.java:195)
at org.springframework.cloud.bootstrap.BootstrapApplicationListener.onApplicationEvent(BootstrapApplicationListener.java:114)
at org.springframework.cloud.bootstrap.BootstrapApplicationListener.onApplicationEvent(BootstrapApplicationListener.java:77)
at org.springframework.context.event.SimpleApplicationEventMulticaster.doInvokeListener(SimpleApplicationEventMulticaster.java:176)
at org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:169)
at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:143)
at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:131)
at org.springframework.boot.context.event.EventPublishingRunListener.environmentPrepared(EventPublishingRunListener.java:85)
at org.springframework.boot.SpringApplicationRunListeners.lambda$environmentPrepared$2(SpringApplicationRunListeners.java:66)
at java.util.ArrayList.forEach(ArrayList.java:1259)
at org.springframework.boot.SpringApplicationRunListeners.doWithListeners(SpringApplicationRunListeners.java:120)
at org.springframework.boot.SpringApplicationRunListeners.doWithListeners(SpringApplicationRunListeners.java:114)
at org.springframework.boot.SpringApplicationRunListeners.environmentPrepared(SpringApplicationRunListeners.java:65)
at org.springframework.boot.SpringApplication.prepareEnvironment(SpringApplication.java:343)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:301)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1317)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1306)
at com.oceanbase.ocp.OcpExpressApplication.main(OcpExpressApplication.java:18)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:49)
at org.springframework.boot.loader.Launcher.launch(Launcher.java:107)
at org.springframework.boot.loader.Launcher.launch(Launcher.java:58)
at org.springframework.boot.loader.PropertiesLauncher.main(PropertiesLauncher.java:467)
Caused by: java.sql.SQLException: execute sql task failed. task: ocp_metric_expr_config, sql:INSERT INTO ocp_metric_expr_config
(metric
,expr
) VALUES (‘active_memstore_percent’,‘100 * sum(ob_sysstat{stat_id=“130000”,@LABELS}) by (@GBLABELS) / sum(ob_sysstat{stat_id=“130002”,@LABELS}) by (@GBLABELS)’) ON DUPLICATE KEY UPDATE expr
=‘100 * sum(ob_sysstat{stat_id=“130000”,@LABELS}) by (@GBLABELS) / sum(ob_sysstat{stat_id=“130002”,@LABELS}) by (@GBLABELS)’
at com.oceanbase.ocp.bootstrap.db.DbInitializer.executeSqlTask(DbInitializer.java:216)
… 49 more
Caused by: java.sql.SQLNonTransientConnectionException: (conn=397) Connection timed out
at com.oceanbase.jdbc.internal.util.exceptions.ExceptionFactory.createException(ExceptionFactory.java:120)
at com.oceanbase.jdbc.internal.util.exceptions.ExceptionFactory.create(ExceptionFactory.java:200)
at com.oceanbase.jdbc.OceanBaseStatement.executeBatchExceptionEpilogue(OceanBaseStatement.java:370)
at com.oceanbase.jdbc.OceanBaseStatement.executeBatch(OceanBaseStatement.java:1487)
at com.oceanbase.ocp.bootstrap.util.SQLUtils.executeBatch(SQLUtils.java:99)
at com.oceanbase.ocp.bootstrap.db.DbInitializer.lambda$new$0(DbInitializer.java:93)
at com.oceanbase.ocp.bootstrap.db.DbInitializer.executeSqlTask(DbInitializer.java:203)
… 49 more
Caused by: com.oceanbase.jdbc.internal.util.exceptions.OceanBaseSqlException: Connection timed out
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.exceptionWithQuery(AbstractQueryProtocol.java:188)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.executeBatchAggregateSemiColon(AbstractQueryProtocol.java:1084)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.executeBatchStmt(AbstractQueryProtocol.java:943)
at com.oceanbase.jdbc.OceanBaseStatement.internalBatchExecution(OceanBaseStatement.java:1544)
at com.oceanbase.jdbc.OceanBaseStatement.executeBatch(OceanBaseStatement.java:1484)
… 52 more
Caused by: java.sql.SQLNonTransientConnectionException: Read timed out
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.handleIoException(AbstractQueryProtocol.java:2361)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.readPacket(AbstractQueryProtocol.java:1839)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.getResult(AbstractQueryProtocol.java:1748)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.executeBatchAggregateSemiColon(AbstractQueryProtocol.java:1080)
… 55 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at java.io.FilterInputStream.read(FilterInputStream.java:133)
at com.oceanbase.jdbc.internal.io.input.ReadAheadBufferedStream.fillBuffer(ReadAheadBufferedStream.java:148)
at com.oceanbase.jdbc.internal.io.input.ReadAheadBufferedStream.read(ReadAheadBufferedStream.java:121)
at com.oceanbase.jdbc.internal.io.input.StandardPacketInputStream.getPacketArray(StandardPacketInputStream.java:247)
at com.oceanbase.jdbc.internal.io.input.StandardPacketInputStream.getPacket(StandardPacketInputStream.java:217)
at com.oceanbase.jdbc.internal.protocol.AbstractQueryProtocol.readPacket(AbstractQueryProtocol.java:1837)
… 57 more