【 使用环境 】生产环境
【 OB or 其他组件 】ocp obproxy
【 使用版本 】
【问题描述】obproxy宕机,ocp上显示最后可用时间和当前可用时间,相差5分钟
【复现路径】问题出现前后相关操作
【附件及日志】
全部日志
2025-03-10 14:54:32.887 INFO 17347 — [pool-manual-subtask-executor14,1b5431261f6a46d2,9a991663de50] c.o.o.c.t.e.runner.JavaSubtaskRunner : Run subtask, id=114215, context=Context{parallelIdx=0, stringMap={former_obproxy_cluster_status=UNLOCK, subtask_splitter=obproxy_server_ids, ocpagent_service_name=proxy_agent, target_obproxy_status=STOPPED, task_instance_id=109664, target_operate_status=NORMAL, task_operation=execute, latest_execution_start_time=2025-03-10T14:54:32.802+08:00, sub_task_instance_name=Inactive agent modules, sub_task_instance_id=114215, obproxy_cluster_id=2, target_obproxy_cluster_status=UNLOCK}, listMap={obproxy_server_ids=[6], host_ids=[2]}}, executor=10.38.36.237
2025-03-10 14:54:32.891 INFO 17347 — [pool-manual-subtask-executor14,1b5431261f6a46d2,9a991663de50] c.o.o.s.t.b.o.InactiveAgentModuleTask : inactive agent modules begin
2025-03-10 14:54:32.952 INFO 17347 — [pool-manual-subtask-executor14,1b5431261f6a46d2,9a991663de50] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=2
2025-03-10 14:54:32.969 INFO 17347 — [pool-manual-subtask-executor14,1b5431261f6a46d2,9a991663de50] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 2
2025-03-10 14:54:33.101 INFO 17347 — [pool-manual-subtask-executor14,1b5431261f6a46d2,9a991663de50] c.o.o.c.agent.HostAgentServiceImpl : get hostServiceType OB_PROXY, exporters {http://10.38.36.239:62889/metrics/obproxy=1, http://10.38.36.239:62889/metrics/node/obproxy=1}
2025-03-10 14:54:33.129 INFO 17347 — [pool-manual-subtask-executor14,1b5431261f6a46d2,9a991663de50] c.o.o.c.agent.HostAgentServiceImpl : Inactiving OCP monitor module, host=2, mode=OB_PROXY_DEPLOYED
2025-03-10 14:54:33.142 INFO 17347 — [pool-manual-subtask-executor14,1b5431261f6a46d2,9a991663de50] c.o.o.c.agent.HostAgentServiceImpl : Finding OCP agent: hostId=2
2025-03-10 14:54:33.149 INFO 17347 — [pool-manual-subtask-executor14,1b5431261f6a46d2,9a991663de50] c.o.o.c.a.p.HostAgentProcessServiceImpl : Getting all OCP agent processes on host 2
2025-03-10 14:54:33.198 INFO 17347 — [pool-manual-subtask-executor14,1b5431261f6a46d2,9a991663de50] c.o.o.e.internal.template.HttpTemplate : POST request to agent, url:http://10.38.36.239:62888/api/v1/module/config/update, request body:UpdateAgentConfigRequest(agentHome=null, configs=[(monagent.pipeline.obproxy.status=inactive)]), params:null
2025-03-10 14:54:33.206 ERROR 17347 — [pool-manual-subtask-executor14,1b5431261f6a46d2,9a991663de50] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : [AgentClient]:http request is failed, response:Authentication failed for wrong digest
com.oceanbase.ocp.executor.exception.HttpRequestFailedException: [AgentClient]:http request is failed, response:Authentication failed for wrong digest
at com.oceanbase.ocp.executor.internal.template.HttpTemplate.checkSuccess(HttpTemplate.java:476)
at com.oceanbase.ocp.executor.internal.template.HttpTemplate.doPost(HttpTemplate.java:286)
at com.oceanbase.ocp.executor.internal.template.HttpTemplate.post(HttpTemplate.java:114)
at com.oceanbase.ocp.executor.executor.AgentExecutor.updateAgentConfig(AgentExecutor.java:185)
at com.oceanbase.ocp.compute.agent.manager.HostAgentConfigManagerImpl.configAgent(HostAgentConfigManagerImpl.java:467)
at com.oceanbase.ocp.compute.agent.manager.HostAgentConfigManagerImpl.configAgent(HostAgentConfigManagerImpl.java:473)
at com.oceanbase.ocp.compute.agent.manager.HostAgentConfigManagerImpl.stopObProxyMonitor(HostAgentConfigManagerImpl.java:432)
at com.oceanbase.ocp.compute.agent.manager.HostAgentConfigManagerImpl$$FastClassBySpringCGLIB$$798eb4cd.invoke()
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
at org.springframework.aop.framework.CglibAopProxy.invokeMethod(CglibAopProxy.java:386)
at org.springframework.aop.framework.CglibAopProxy.access$000(CglibAopProxy.java:85)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:704)
at com.oceanbase.ocp.compute.agent.manager.HostAgentConfigManagerImpl$$EnhancerBySpringCGLIB$$a21ae10f.stopObProxyMonitor()
at com.oceanbase.ocp.compute.agent.HostAgentServiceImpl.inactiveObProxyModule(HostAgentServiceImpl.java:831)
at com.oceanbase.ocp.compute.agent.HostAgentServiceImpl.InactiveAgentModule(HostAgentServiceImpl.java:437)
at com.oceanbase.ocp.compute.agent.HostAgentServiceImpl$$FastClassBySpringCGL
IB$$e7947c56.invoke()
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
at org.springframework.aop.framework.CglibAopProxy.invokeMethod(CglibAopProxy.java:386)
at org.springframework.aop.framework.CglibAopProxy.access$000(CglibAopProxy.java:85)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:704)
at com.oceanbase.ocp.compute.agent.HostAgentServiceImpl$$EnhancerBySpringCGLIB$$bcbcb54a.InactiveAgentModule()
at com.oceanbase.ocp.service.task.business.obagent.InactiveAgentModuleTask.run(InactiveAgentModuleTask.java:54)
at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.execute(JavaSubtaskRunner.java:64)
at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:32)
at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26)
at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76)
at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:203)
at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:197)
at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:134)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Set state for subtask: 114215, operation:EXECUTE, state: FAILED
