[2025-01-14 17:00:00.504] [WARN] [refresh_regions] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:00:00.506] [WARN] [refresh_regions] 43.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 44.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:00:00.508] [ERROR] [grpc-default-executor-615] onErrorEventHandle : region_id: 132931593 error { not_leader { region_id: 132931593 } } request_id: 153 [2025-01-14 17:00:00.509] [ERROR] [grpc-default-executor-615] region CDC error: region: 132931593, resolvedTs:455303540096106502 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 132931593 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:00.509] [ERROR] [grpc-default-executor-615] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 132931593 } , regionId: 132931593 , resolvedTs : 455303540096106502 [2025-01-14 17:00:00.513] [ERROR] [grpc-default-executor-672] onErrorEventHandle : region_id: 139136520 error { not_leader { region_id: 139136520 } } request_id: 52 [2025-01-14 17:00:00.513] [ERROR] [grpc-default-executor-672] region CDC error: region: 139136520, resolvedTs:455303545115639813 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136520 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:00.513] [ERROR] [grpc-default-executor-672] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136520 } , regionId: 139136520 , resolvedTs : 455303545115639813 [2025-01-14 17:00:07.754] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:00:45.634] [ERROR] [grpc-default-executor-617] onErrorEventHandle : region_id: 138055782 error { not_leader { region_id: 138055782 } } request_id: 78 [2025-01-14 17:00:45.635] [ERROR] [grpc-default-executor-617] region CDC error: region: 138055782, resolvedTs:455303559861502024 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138055782 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:45.635] [ERROR] [grpc-default-executor-617] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138055782 } , regionId: 138055782 , resolvedTs : 455303559861502024 [2025-01-14 17:00:46.374] [ERROR] [grpc-default-executor-641] onErrorEventHandle : region_id: 130958217 error { not_leader { region_id: 130958217 } } request_id: 58 [2025-01-14 17:00:46.374] [ERROR] [grpc-default-executor-641] region CDC error: region: 130958217, resolvedTs:455303560123384015 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 130958217 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:46.374] [ERROR] [grpc-default-executor-641] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 130958217 } , regionId: 130958217 , resolvedTs : 455303560123384015 [2025-01-14 17:00:49.652] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:00:50.904] [ERROR] [grpc-default-executor-641] onErrorEventHandle : region_id: 138780556 error { not_leader { region_id: 138780556 } } request_id: 137 [2025-01-14 17:00:50.905] [ERROR] [grpc-default-executor-641] region CDC error: region: 138780556, resolvedTs:455303561434104160 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138780556 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:50.905] [ERROR] [grpc-default-executor-641] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138780556 } , regionId: 138780556 , resolvedTs : 455303561434104160 [2025-01-14 17:00:51.668] [ERROR] [grpc-default-executor-641] onErrorEventHandle : region_id: 130610857 error { not_leader { region_id: 130610857 } } request_id: 83 [2025-01-14 17:00:51.668] [ERROR] [grpc-default-executor-641] region CDC error: region: 130610857, resolvedTs:455303561434104160 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 130610857 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:51.668] [ERROR] [grpc-default-executor-641] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 130610857 } , regionId: 130610857 , resolvedTs : 455303561434104160 [2025-01-14 17:00:51.786] [ERROR] [grpc-default-executor-641] onErrorEventHandle : region_id: 131011641 error { not_leader { region_id: 131011641 } } request_id: 159 [2025-01-14 17:00:51.786] [ERROR] [grpc-default-executor-641] region CDC error: region: 131011641, resolvedTs:455303561434104160 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131011641 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:51.786] [ERROR] [grpc-default-executor-641] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131011641 } , regionId: 131011641 , resolvedTs : 455303561434104160 [2025-01-14 17:00:52.902] [ERROR] [grpc-default-executor-617] onErrorEventHandle : region_id: 131623409 error { not_leader { region_id: 131623409 } } request_id: 33 [2025-01-14 17:00:52.903] [ERROR] [grpc-default-executor-617] region CDC error: region: 131623409, resolvedTs:455303561696247984 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131623409 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:52.903] [ERROR] [grpc-default-executor-617] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131623409 } , regionId: 131623409 , resolvedTs : 455303561696247984 [2025-01-14 17:00:52.960] [ERROR] [grpc-default-executor-617] onErrorEventHandle : region_id: 132620181 error { not_leader { region_id: 132620181 } } request_id: 93 [2025-01-14 17:00:52.961] [ERROR] [grpc-default-executor-617] region CDC error: region: 132620181, resolvedTs:455303561696247984 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 132620181 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:52.961] [ERROR] [grpc-default-executor-617] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 132620181 } , regionId: 132620181 , resolvedTs : 455303561696247984 [2025-01-14 17:00:53.984] [ERROR] [grpc-default-executor-641] onErrorEventHandle : region_id: 137492087 error { not_leader { region_id: 137492087 } } request_id: 55 [2025-01-14 17:00:53.984] [ERROR] [grpc-default-executor-641] region CDC error: region: 137492087, resolvedTs:455303562220535887 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 137492087 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:53.984] [ERROR] [grpc-default-executor-641] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 137492087 } , regionId: 137492087 , resolvedTs : 455303562220535887 [2025-01-14 17:00:55.488] [ERROR] [grpc-default-executor-641] onErrorEventHandle : region_id: 133509565 error { not_leader { region_id: 133509565 } } request_id: 59 [2025-01-14 17:00:55.488] [ERROR] [grpc-default-executor-641] region CDC error: region: 133509565, resolvedTs:455303562220535887 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 133509565 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:55.488] [ERROR] [grpc-default-executor-641] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 133509565 } , regionId: 133509565 , resolvedTs : 455303562220535887 [2025-01-14 17:00:56.061] [ERROR] [grpc-default-executor-641] onErrorEventHandle : region_id: 139136652 error { not_leader { region_id: 139136652 } } request_id: 136 [2025-01-14 17:00:56.062] [ERROR] [grpc-default-executor-641] region CDC error: region: 139136652, resolvedTs:455303562220535887 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136652 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:56.062] [ERROR] [grpc-default-executor-641] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136652 } , regionId: 139136652 , resolvedTs : 455303562220535887 [2025-01-14 17:00:56.670] [ERROR] [grpc-default-executor-641] onErrorEventHandle : region_id: 133320645 error { not_leader { region_id: 133320645 } } request_id: 41 [2025-01-14 17:00:56.671] [ERROR] [grpc-default-executor-641] region CDC error: region: 133320645, resolvedTs:455303562220535887 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 133320645 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:00:56.671] [ERROR] [grpc-default-executor-641] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 133320645 } , regionId: 133320645 , resolvedTs : 455303562220535887 [2025-01-14 17:01:01.654] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:01:01.985] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:01:11.482] [ERROR] [grpc-default-executor-617] onErrorEventHandle : region_id: 139136592 error { not_leader { region_id: 139136592 } } request_id: 21 [2025-01-14 17:01:11.483] [ERROR] [grpc-default-executor-617] region CDC error: region: 139136592, resolvedTs:455303566598340633 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136592 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:01:11.483] [ERROR] [grpc-default-executor-617] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136592 } , regionId: 139136592 , resolvedTs : 455303566598340633 [2025-01-14 17:01:19.957] [INFO] [region_error_worker] remove regions: [130610857, 139136520, 132931593, 137492087, 130958217, 139136652, 133509565, 138780556, 131623409, 132620181, 131011641, 133320645, 139136592, 138055782] [2025-01-14 17:01:19.957] [INFO] [region_error_worker] close (region: 130610857) [2025-01-14 17:01:19.957] [INFO] [region_error_worker] terminated (region: 130610857) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 139136520) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 139136520) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 132931593) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 132931593) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 137492087) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 137492087) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 130958217) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 130958217) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 139136652) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 139136652) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 133509565) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 133509565) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 138780556) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 138780556) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 131623409) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 131623409) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 132620181) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 132620181) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 131011641) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 131011641) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 133320645) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 133320645) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 139136592) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 139136592) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] close (region: 138055782) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] terminated (region: 138055782) [2025-01-14 17:01:19.958] [INFO] [region_error_worker] regions error and restart [130610857, 139136520, 132931593, 137492087, 130958217, 139136652, 133509565, 138780556, 131623409, 132620181, 131011641, 133320645, 139136592, 138055782] [2025-01-14 17:01:29.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:01:30.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:01:40.080] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:01:40.080] [WARN] [region_error_worker] 44.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 46.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:01:40.080] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:01:40.083] [ERROR] [grpc-default-executor-717] onErrorEventHandle : region_id: 131402865 error { not_leader { region_id: 131402865 } } request_id: 116 [2025-01-14 17:01:40.083] [ERROR] [grpc-default-executor-717] region CDC error: region: 131402865, resolvedTs:455303571631767558 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131402865 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:01:40.083] [ERROR] [grpc-default-executor-717] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131402865 } , regionId: 131402865 , resolvedTs : 455303571631767558 [2025-01-14 17:01:43.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:01:44.752] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:01:50.390] [ERROR] [grpc-default-executor-530] onErrorEventHandle : region_id: 139065664 error { not_leader { region_id: 139065664 } } request_id: 8 [2025-01-14 17:01:50.391] [ERROR] [grpc-default-executor-530] region CDC error: region: 139065664, resolvedTs:455303577005457411 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139065664 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:01:50.391] [ERROR] [grpc-default-executor-530] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139065664 } , regionId: 139065664 , resolvedTs : 455303577005457411 [2025-01-14 17:01:53.581] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:01:53.947] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:01:58.655] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:01:59.327] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:02:12.604] [ERROR] [grpc-default-executor-639] onErrorEventHandle : region_id: 138071575 error { not_leader { region_id: 138071575 } } request_id: 36 [2025-01-14 17:02:12.605] [ERROR] [grpc-default-executor-639] region CDC error: region: 138071575, resolvedTs:455303582772625413 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138071575 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:02:12.605] [ERROR] [grpc-default-executor-639] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138071575 } , regionId: 138071575 , resolvedTs : 455303582772625413 [2025-01-14 17:02:15.341] [ERROR] [grpc-default-executor-639] onErrorEventHandle : region_id: 139135836 error { epoch_not_match { } } request_id: 19 [2025-01-14 17:02:15.341] [ERROR] [grpc-default-executor-639] region CDC error: region: 139135836, resolvedTs:455303583559057436 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:02:15.341] [ERROR] [grpc-default-executor-639] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139135836 , resolvedTs : 455303583559057436 [2025-01-14 17:02:15.346] [ERROR] [grpc-default-executor-639] onErrorEventHandle : region_id: 139124632 error { epoch_not_match { } } request_id: 145 [2025-01-14 17:02:15.346] [ERROR] [grpc-default-executor-639] region CDC error: region: 139124632, resolvedTs:455303583559057436 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:02:15.346] [ERROR] [grpc-default-executor-639] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139124632 , resolvedTs : 455303583559057436 [2025-01-14 17:02:26.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:02:27.531] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:02:38.251] [ERROR] [grpc-default-executor-713] onErrorEventHandle : region_id: 139136644 error { not_leader { region_id: 139136644 } } request_id: 143 [2025-01-14 17:02:38.252] [ERROR] [grpc-default-executor-713] region CDC error: region: 139136644, resolvedTs:455303589392285702 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136644 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:02:38.252] [ERROR] [grpc-default-executor-713] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136644 } , regionId: 139136644 , resolvedTs : 455303589392285702 [2025-01-14 17:02:40.538] [INFO] [TidbEventSorter] lastHeartBeatTs : 455303108231168001 , lastSendHeartBeatTs : 1736844140243 , minResolvedTs : 1736843522 455303108231168001 , {139128648=455303590112919560, 139131720=455303590112919558, 139136584=455303590112919560, 139135816=455303301570756627, 139136588=455303293682581564, 137464374=455303590112919560, 139135820=455303519373099009, 139065664=455303577005457411, 139135808=455303334325649479, 139135812=455303355611742215, 139136580=455303307875319809, 139135832=455303351391223831, 139136600=455303590112919560, 139143768=455303590112919560, 134760989=455303590112919560, 138058799=455303590112919560, 134210281=455303477156380674, 139136604=455303590112919560, 139135836=455303583559057436, 139135824=455303362099806209, 139136592=455303566598340633, 139133520=455303300260298756, 137463084=455303590112919560, 139136596=455303590112919560, 139135828=455303535455633419, 131623161=455303275303927811, 139131752=455303455383748616, 139140200=455303590112919560, 139133800=455303419771486211, 139136616=455303271319339011, 131623165=455303330170667011, 139136620=455303345099767840, 131623409=455303561696247984, 133206489=455303308320964616, 139136608=455303336475230278, 138071575=455303582772625413, 139136612=455303282552209494, 130992353=455303362099806209, 139136632=455303590112919560, 130989561=455303350119825441, 133320645=455303562220535887, 139136636=455303477156380674, 130820597=455303590112919560, 137533705=455303477561131010, 139136624=455303286602334216, 134273597=455303292631646280, 137533711=455303590112919560, 139136628=455303508310622234, 139133812=455303590112919560, 131545749=455303590112919558, 131045517=455303590112919560, 139136520=455303545115639813, 139135752=455303407332753409, 129007605=455303276614647811, 137492087=455303562220535887, 139135756=455303423441502235, 139136524=455303356136030209, 130958217=455303560123384015, 133509565=455303562220535887, 139136512=455303590112919560, 139135744=455303359635652611, 131765393=455303362099806209, 139135748=455303407424503809, 137533309=455303590112919560, 139136516=455303272931524629, 138100330=455303358036574212, 132648121=455303477156380674, 139135768=455303423441502235, 138100334=455303271974699013, 139135772=455303455383748616, 127235020=455303590112919558, 127518659=455303477156380674, 139120144=455303310234615809, 133425833=455303324102295553, 139135760=455303423441502235, 139136528=455303515414724613, 137473898=455303590112919560, 138055782=455303559861502024, 138767896=455303518062379018, 139136532=455303590112919560, 139135764=455303423441502235, 131619513=455303336214659117, 130610857=455303561434104160, 138776612=455303590112919560, 138827044=455303511993745418, 138081885=455303284138180610, 139135788=455303333853790255, 131686577=455303444111556610, 139135776=455303315949355042, 127596788=455303303405764609, 139135780=455303455383748616, 126613476=455303348717355012, 132620181=455303561696247984, 137997899=455303430270353415, 132640149=455303288070864966, 139133240=455303590112919560, 139135800=455303455383748616, 132587665=455303590112919558, 139135804=455303519386206210, 137544777=455303533884604422, 130514245=455303477561131010, 139100104=455303425709047819, 139136712=455303477156380674, 131641437=455303590112919560, 139124940=455303279236087860, 139136716=455303590112919560, 131401309=455303590112919558, 135574429=455303477561131010, 129670449=455303590112919560, 139136704=455303361929412614, 139000004=455303590112919560, 139136708=455303425538916377, 137463720=455303590112919558, 139136720=455303529754001413, 136705209=455303590112919560, 131402865=455303571631767558, 131054953=455303590112919560, 137512084=455303275946180618, 139132896=455303303838564369, 131687285=455303536795713543, 138007444=455303590112919560, 132795205=455303295266193443, 139133412=455303590112919560, 131905645=455303590112919560, 130763641=455303287808196672, 139136504=455303537461035011, 139136508=455303361405124613, 131422573=455303590112919558, 130725745=455303362099806209, 139136496=455303590112919560, 131844197=455303430532497453, 139136500=455303427913940993, 137798644=455303590112919558, 139136648=455303320209195025, 126675016=455303477156380674, 139136652=455303562220535887, 138780556=455303561434104160, 138906508=455303590112919560, 139136640=455303477156380674, 130825217=455303333853790255, 139131780=455303590112919558, 131739669=455303590112919560, 139136644=455303589392285702, 137470945=455303271451197441, 139124632=455303583559057436, 137533921=455303324627107846, 130111765=455303590112919560, 126799444=455303362099806209, 139136656=455303279761686589, 127663168=455303362099806209, 139124628=455303590112919560, 133944357=455303108231168001, 132931593=455303540096106502, 139136684=455303477156380674, 131729469=455303477156380674, 131710773=455303477156380674, 139136696=455303590112919560, 137797824=455303308320964616, 131011641=455303561434104160, 139136700=455303358298718214, 139136688=455303534407057413, 139134128=455303279381315631, 139136692=455303355677278224} , [455303108231168001, 455303271319339011, 455303271451197441, 455303271974699013, 455303272931524629, 455303275303927811, 455303275946180618, 455303276614647811, 455303279236087860, 455303279381315631, 455303279761686589, 455303282552209494, 455303284138180610, 455303286602334216, 455303287808196672, 455303288070864966, 455303292631646280, 455303293682581564, 455303295266193443, 455303300260298756, 455303301570756627, 455303303405764609, 455303303838564369, 455303307875319809, 455303308320964616 x 2, 455303310234615809, 455303315949355042, 455303320209195025, 455303324102295553, 455303324627107846, 455303330170667011, 455303333853790255 x 2, 455303334325649479, 455303336214659117, 455303336475230278, 455303345099767840, 455303348717355012, 455303350119825441, 455303351391223831, 455303355611742215, 455303355677278224, 455303356136030209, 455303358036574212, 455303358298718214, 455303359635652611, 455303361405124613, 455303361929412614, 455303362099806209 x 6, 455303407332753409, 455303407424503809, 455303419771486211, 455303423441502235 x 4, 455303425538916377, 455303425709047819, 455303427913940993, 455303430270353415, 455303430532497453, 455303444111556610, 455303455383748616 x 4, 455303477156380674 x 10, 455303477561131010 x 3, 455303508310622234, 455303511993745418, 455303515414724613, 455303518062379018, 455303519373099009, 455303519386206210, 455303529754001413, 455303533884604422, 455303534407057413, 455303535455633419, 455303536795713543, 455303537461035011, 455303540096106502, 455303545115639813, 455303559861502024, 455303560123384015, 455303561434104160 x 3, 455303561696247984 x 2, 455303562220535887 x 4, 455303566598340633, 455303571631767558, 455303577005457411, 455303582772625413, 455303583559057436 x 2, 455303589392285702, 455303590112919558 x 9, 455303590112919560 x 37] [2025-01-14 17:02:41.263] [INFO] [grpc-default-executor-714] handle resolvedTs: 1736845361232, regionId: 139131720 [2025-01-14 17:02:41.263] [INFO] [grpc-default-executor-718] handle resolvedTs: 1736845361232, regionId: 127235020 [2025-01-14 17:02:41.264] [INFO] [grpc-default-executor-639] handle resolvedTs: 1736845361232, regionId: 132587665 [2025-01-14 17:02:41.264] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361232, regionId: 139131780 [2025-01-14 17:02:41.264] [INFO] [grpc-default-executor-639] handle resolvedTs: 1736845361232, regionId: 137463720 [2025-01-14 17:02:41.264] [INFO] [grpc-default-executor-639] handle resolvedTs: 1736845361232, regionId: 131422573 [2025-01-14 17:02:41.264] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361232, regionId: 137798644 [2025-01-14 17:02:41.264] [INFO] [grpc-default-executor-726] handle resolvedTs: 1736845361232, regionId: 131401309 [2025-01-14 17:02:41.264] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361232, regionId: 131545749 [2025-01-14 17:02:41.491] [INFO] [grpc-default-executor-726] handle resolvedTs: 1736845361482, regionId: 134760989 [2025-01-14 17:02:41.491] [INFO] [grpc-default-executor-714] handle resolvedTs: 1736845361482, regionId: 137533711 [2025-01-14 17:02:41.491] [INFO] [grpc-default-executor-639] handle resolvedTs: 1736845361482, regionId: 139133240 [2025-01-14 17:02:41.491] [INFO] [grpc-default-executor-726] handle resolvedTs: 1736845361482, regionId: 138007444 [2025-01-14 17:02:41.491] [INFO] [grpc-default-executor-718] handle resolvedTs: 1736845361482, regionId: 139000004 [2025-01-14 17:02:41.491] [INFO] [grpc-default-executor-639] handle resolvedTs: 1736845361482, regionId: 138776612 [2025-01-14 17:02:41.491] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361482, regionId: 129670449 [2025-01-14 17:02:41.491] [INFO] [grpc-default-executor-639] handle resolvedTs: 1736845361482, regionId: 139136632 [2025-01-14 17:02:41.491] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361482, regionId: 130111765 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-726] handle resolvedTs: 1736845361482, regionId: 139133812 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-714] handle resolvedTs: 1736845361482, regionId: 139140200 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-718] handle resolvedTs: 1736845361482, regionId: 139136584 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361482, regionId: 137533309 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-726] handle resolvedTs: 1736845361482, regionId: 131045517 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361482, regionId: 138906508 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-726] handle resolvedTs: 1736845361482, regionId: 139136600 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361482, regionId: 139133412 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-714] handle resolvedTs: 1736845361482, regionId: 137463084 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-726] handle resolvedTs: 1736845361482, regionId: 131641437 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361482, regionId: 137473898 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-530] handle resolvedTs: 1736845361482, regionId: 130820597 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361482, regionId: 139136596 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-714] handle resolvedTs: 1736845361482, regionId: 139143768 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-530] handle resolvedTs: 1736845361482, regionId: 131905645 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-718] handle resolvedTs: 1736845361482, regionId: 136705209 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361482, regionId: 137464374 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-726] handle resolvedTs: 1736845361482, regionId: 139124628 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-714] handle resolvedTs: 1736845361482, regionId: 139136532 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-726] handle resolvedTs: 1736845361482, regionId: 139136696 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-718] handle resolvedTs: 1736845361482, regionId: 139136496 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-726] handle resolvedTs: 1736845361482, regionId: 138058799 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-714] handle resolvedTs: 1736845361482, regionId: 139136604 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-530] handle resolvedTs: 1736845361482, regionId: 131739669 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-718] handle resolvedTs: 1736845361482, regionId: 139136716 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-639] handle resolvedTs: 1736845361482, regionId: 139128648 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361482, regionId: 131054953 [2025-01-14 17:02:41.492] [INFO] [grpc-default-executor-713] handle resolvedTs: 1736845361482, regionId: 139136512 [2025-01-14 17:02:50.772] [INFO] [grpc-default-executor-639] ignore DDL : start_ts: 455303592865169439 type: PREWRITE op_type: PUT key: "mDB:51\000\000\000\374\000\000\000\000\000\000\000hTID:63\000\000\375" value: "6746993651" old_value: "6746963651" [2025-01-14 17:02:50.773] [INFO] [grpc-default-executor-639] ignore DDL : start_ts: 455303592865169439 commit_ts: 455303592865169440 type: COMMIT op_type: PUT key: "mDB:51\000\000\000\374\000\000\000\000\000\000\000hTID:63\000\000\375" value: "6746993651" [2025-01-14 17:03:01.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:03:01.765] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:03:09.630] [ERROR] [grpc-default-executor-691] onErrorEventHandle : region_id: 139140200 error { not_leader { region_id: 139140200 } } request_id: 28 [2025-01-14 17:03:09.630] [ERROR] [grpc-default-executor-691] region CDC error: region: 139140200, resolvedTs:455303597623083074 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139140200 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:03:09.630] [ERROR] [grpc-default-executor-691] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139140200 } , regionId: 139140200 , resolvedTs : 455303597623083074 [2025-01-14 17:03:17.872] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:03:18.184] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:03:19.957] [INFO] [region_error_worker] remove regions: [139124632, 139140200, 131402865, 139135836, 139065664, 138071575, 139136644] [2025-01-14 17:03:19.958] [INFO] [region_error_worker] close (region: 139124632) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] terminated (region: 139124632) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] close (region: 139140200) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] terminated (region: 139140200) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] close (region: 131402865) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] terminated (region: 131402865) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] close (region: 139135836) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] terminated (region: 139135836) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] close (region: 139065664) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] terminated (region: 139065664) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] close (region: 138071575) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] terminated (region: 138071575) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] close (region: 139136644) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] terminated (region: 139136644) [2025-01-14 17:03:19.958] [INFO] [region_error_worker] regions error and restart [139124632, 139140200, 131402865, 139135836, 139065664, 138071575, 139136644] [2025-01-14 17:03:40.079] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:03:40.229] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:03:40.229] [WARN] [region_error_worker] 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 46.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 47.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:03:40.229] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:03:40.233] [ERROR] [grpc-default-executor-766] onErrorEventHandle : region_id: 136705209 error { not_leader { region_id: 136705209 } } request_id: 115 [2025-01-14 17:03:40.233] [ERROR] [grpc-default-executor-766] region CDC error: region: 136705209, resolvedTs:455303604084932619 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 136705209 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:03:40.233] [ERROR] [grpc-default-executor-766] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 136705209 } , regionId: 136705209 , resolvedTs : 455303604084932619 [2025-01-14 17:03:40.740] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:03:51.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:03:52.102] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:04:01.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:04:01.077] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:04:10.424] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:04:10.463] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:04:12.404] [ERROR] [grpc-default-executor-751] onErrorEventHandle : region_id: 137473898 error { not_leader { region_id: 137473898 } } request_id: 77 [2025-01-14 17:04:12.404] [ERROR] [grpc-default-executor-751] region CDC error: region: 137473898, resolvedTs:455303614125572097 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 137473898 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:04:12.404] [ERROR] [grpc-default-executor-751] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 137473898 } , regionId: 137473898 , resolvedTs : 455303614125572097 [2025-01-14 17:04:14.420] [ERROR] [grpc-default-executor-766] onErrorEventHandle : region_id: 139136604 error { not_leader { region_id: 139136604 } } request_id: 18 [2025-01-14 17:04:14.420] [ERROR] [grpc-default-executor-766] region CDC error: region: 139136604, resolvedTs:455303614636228662 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136604 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:04:14.420] [ERROR] [grpc-default-executor-766] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136604 } , regionId: 139136604 , resolvedTs : 455303614636228662 [2025-01-14 17:04:17.413] [ERROR] [grpc-default-executor-766] onErrorEventHandle : region_id: 139131780 error { epoch_not_match { } } request_id: 141 [2025-01-14 17:04:17.413] [ERROR] [grpc-default-executor-766] region CDC error: region: 139131780, resolvedTs:455303615357124622 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:04:17.413] [ERROR] [grpc-default-executor-766] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139131780 , resolvedTs : 455303615357124622 [2025-01-14 17:04:17.554] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:04:17.954] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:04:19.264] [ERROR] [grpc-default-executor-713] onErrorEventHandle : region_id: 131045517 error { not_leader { region_id: 131045517 } } request_id: 51 [2025-01-14 17:04:19.264] [ERROR] [grpc-default-executor-713] region CDC error: region: 131045517, resolvedTs:455303615960317953 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131045517 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:04:19.264] [ERROR] [grpc-default-executor-713] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131045517 } , regionId: 131045517 , resolvedTs : 455303615960317953 [2025-01-14 17:04:24.080] [ERROR] [grpc-default-executor-713] onErrorEventHandle : region_id: 137464374 error { not_leader { region_id: 137464374 } } request_id: 6 [2025-01-14 17:04:24.080] [ERROR] [grpc-default-executor-713] region CDC error: region: 137464374, resolvedTs:455303617270775818 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 137464374 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:04:24.080] [ERROR] [grpc-default-executor-713] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 137464374 } , regionId: 137464374 , resolvedTs : 455303617270775818 [2025-01-14 17:04:27.026] [ERROR] [grpc-default-executor-753] onErrorEventHandle : region_id: 139124628 error { not_leader { region_id: 139124628 } } request_id: 151 [2025-01-14 17:04:27.026] [ERROR] [grpc-default-executor-753] region CDC error: region: 139124628, resolvedTs:455303618057207828 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139124628 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:04:27.026] [ERROR] [grpc-default-executor-753] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139124628 } , regionId: 139124628 , resolvedTs : 455303618057207828 [2025-01-14 17:04:28.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:04:29.268] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:04:34.054] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:04:34.146] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:04:38.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:04:38.865] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:04:43.145] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:04:43.421] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:04:44.009] [ERROR] [grpc-default-executor-759] onErrorEventHandle : region_id: 138058799 error { not_leader { region_id: 138058799 } } request_id: 16 [2025-01-14 17:04:44.009] [ERROR] [grpc-default-executor-759] region CDC error: region: 138058799, resolvedTs:455303622251773957 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138058799 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:04:44.009] [ERROR] [grpc-default-executor-759] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138058799 } , regionId: 138058799 , resolvedTs : 455303622251773957 [2025-01-14 17:04:44.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:04:44.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:04:46.344] [ERROR] [grpc-default-executor-755] onErrorEventHandle : region_id: 131641437 error { not_leader { region_id: 131641437 } } request_id: 104 [2025-01-14 17:04:46.345] [ERROR] [grpc-default-executor-755] region CDC error: region: 131641437, resolvedTs:455303623064158215 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131641437 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:04:46.345] [ERROR] [grpc-default-executor-755] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131641437 } , regionId: 131641437 , resolvedTs : 455303623064158215 [2025-01-14 17:04:49.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:04:49.677] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:04:58.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:04:58.993] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:04.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:04.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:08.379] [ERROR] [grpc-default-executor-759] onErrorEventHandle : region_id: 137463084 error { not_leader { region_id: 137463084 } } request_id: 23 [2025-01-14 17:05:08.380] [ERROR] [grpc-default-executor-759] region CDC error: region: 137463084, resolvedTs:455303628831326231 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 137463084 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:05:08.380] [ERROR] [grpc-default-executor-759] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 137463084 } , regionId: 137463084 , resolvedTs : 455303628831326231 [2025-01-14 17:05:09.999] [INFO] [grpc-default-executor-766] ignore DDL : start_ts: 455303629368721423 type: PREWRITE op_type: PUT key: "mDB:51\000\000\000\374\000\000\000\000\000\000\000hTID:63\000\000\375" value: "6747023651" old_value: "6746993651" [2025-01-14 17:05:10.000] [INFO] [grpc-default-executor-766] ignore DDL : start_ts: 455303629368721423 commit_ts: 455303629368721424 type: COMMIT op_type: PUT key: "mDB:51\000\000\000\374\000\000\000\000\000\000\000hTID:63\000\000\375" value: "6747023651" [2025-01-14 17:05:18.665] [ERROR] [grpc-default-executor-751] onErrorEventHandle : region_id: 139131720 error { epoch_not_match { } } request_id: 2 [2025-01-14 17:05:18.665] [ERROR] [grpc-default-executor-751] region CDC error: region: 139131720, resolvedTs:455303631387754500 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.DelayedClientCall$DelayedListener.onMessage(DelayedClientCall.java:447) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:05:18.665] [ERROR] [grpc-default-executor-751] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139131720 , resolvedTs : 455303631387754500 [2025-01-14 17:05:19.957] [INFO] [region_error_worker] remove regions: [131045517, 139131720, 138058799, 131641437, 139136604, 137464374, 137473898, 136705209, 137463084, 139131780, 139124628] [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 131045517) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 131045517) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 139131720) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 139131720) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 138058799) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 138058799) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 131641437) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 131641437) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 139136604) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 139136604) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 137464374) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 137464374) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 137473898) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 137473898) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 136705209) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 136705209) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 137463084) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 137463084) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 139131780) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 139131780) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] close (region: 139124628) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] terminated (region: 139124628) [2025-01-14 17:05:19.958] [INFO] [region_error_worker] regions error and restart [131045517, 139131720, 138058799, 131641437, 139136604, 137464374, 137473898, 136705209, 137463084, 139131780, 139124628] [2025-01-14 17:05:22.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:22.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:27.454] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:27.517] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:32.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:33.755] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:37.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:38.041] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:38.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:38.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:39.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:40.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:40.165] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:05:40.165] [WARN] [region_error_worker] 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 46.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 47.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:05:40.166] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:05:44.014] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:44.060] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:48.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:49.117] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:54.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:55.055] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:05:59.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:05:59.870] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:03.765] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:04.090] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:05.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:05.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:10.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:10.142] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:14.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:14.897] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:18.248] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:18.497] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:19.989] [ERROR] [grpc-default-executor-753] onErrorEventHandle : region_id: 139136496 error { epoch_not_match { } } request_id: 130 [2025-01-14 17:06:19.989] [ERROR] [grpc-default-executor-753] region CDC error: region: 139136496, resolvedTs:455303647246942209 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:06:19.989] [ERROR] [grpc-default-executor-753] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139136496 , resolvedTs : 455303647246942209 [2025-01-14 17:06:19.989] [ERROR] [grpc-default-executor-792] onErrorEventHandle : region_id: 139136512 error { epoch_not_match { } } request_id: 60 [2025-01-14 17:06:19.991] [ERROR] [grpc-default-executor-792] region CDC error: region: 139136512, resolvedTs:455303647246942209 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:06:19.991] [ERROR] [grpc-default-executor-792] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139136512 , resolvedTs : 455303647246942209 [2025-01-14 17:06:20.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:20.922] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:21.454] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:21.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:22.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:24.210] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:24.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:25.382] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:26.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:26.324] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:30.744] [ERROR] [grpc-default-executor-792] onErrorEventHandle : region_id: 139000004 error { not_leader { region_id: 139000004 } } request_id: 111 [2025-01-14 17:06:30.745] [ERROR] [grpc-default-executor-792] region CDC error: region: 139000004, resolvedTs:455303650392670209 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139000004 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:06:30.745] [ERROR] [grpc-default-executor-792] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139000004 } , regionId: 139000004 , resolvedTs : 455303650392670209 [2025-01-14 17:06:30.754] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:31.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:34.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:34.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:38.367] [ERROR] [grpc-default-executor-792] onErrorEventHandle : region_id: 139136600 error { not_leader { region_id: 139136600 } } request_id: 13 [2025-01-14 17:06:38.367] [ERROR] [grpc-default-executor-792] region CDC error: region: 139136600, resolvedTs:455303652489822232 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136600 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:06:38.367] [ERROR] [grpc-default-executor-792] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136600 } , regionId: 139136600 , resolvedTs : 455303652489822232 [2025-01-14 17:06:39.150] [ERROR] [grpc-default-executor-753] onErrorEventHandle : region_id: 130111765 error { not_leader { region_id: 130111765 } } request_id: 147 [2025-01-14 17:06:39.150] [ERROR] [grpc-default-executor-753] region CDC error: region: 130111765, resolvedTs:455303652489822232 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 130111765 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:06:39.150] [ERROR] [grpc-default-executor-753] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 130111765 } , regionId: 130111765 , resolvedTs : 455303652489822232 [2025-01-14 17:06:40.749] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:40.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:41.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:41.884] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:42.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:43.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:43.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:43.724] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:44.288] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:45.834] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:54.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:55.996] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:06:56.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:06:57.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:01.127] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:02.260] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:02.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:03.368] [ERROR] [grpc-default-executor-753] onErrorEventHandle : region_id: 139136596 error { not_leader { region_id: 139136596 } } request_id: 24 [2025-01-14 17:07:03.369] [ERROR] [grpc-default-executor-753] region CDC error: region: 139136596, resolvedTs:455303658794385410 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136596 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:07:03.369] [ERROR] [grpc-default-executor-753] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136596 } , regionId: 139136596 , resolvedTs : 455303658794385410 [2025-01-14 17:07:03.520] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:04.054] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:05.025] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:05.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:07.022] [ERROR] [grpc-default-executor-770] onErrorEventHandle : region_id: 139136632 error { not_leader { region_id: 139136632 } } request_id: 39 [2025-01-14 17:07:07.022] [ERROR] [grpc-default-executor-770] region CDC error: region: 139136632, resolvedTs:455303659842961417 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136632 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:07:07.022] [ERROR] [grpc-default-executor-770] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139136632 } , regionId: 139136632 , resolvedTs : 455303659842961417 [2025-01-14 17:07:07.054] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:19.957] [INFO] [region_error_worker] remove regions: [139136600, 139136632, 130111765, 139136496, 139136512, 139000004, 139136596] [2025-01-14 17:07:19.958] [INFO] [region_error_worker] close (region: 139136600) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] terminated (region: 139136600) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] close (region: 139136632) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] terminated (region: 139136632) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] close (region: 130111765) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] terminated (region: 130111765) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] close (region: 139136496) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] terminated (region: 139136496) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] close (region: 139136512) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] terminated (region: 139136512) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] close (region: 139000004) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] terminated (region: 139000004) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] close (region: 139136596) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] terminated (region: 139136596) [2025-01-14 17:07:19.958] [INFO] [region_error_worker] regions error and restart [139136600, 139136632, 130111765, 139136496, 139136512, 139000004, 139136596] [2025-01-14 17:07:31.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:31.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:36.200] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:36.455] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:40.176] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:07:40.176] [WARN] [region_error_worker] 46.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 47.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 48.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:07:40.177] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:07:40.177] [ERROR] [grpc-default-executor-816] onErrorEventHandle : region_id: 139136532 error { epoch_not_match { } } request_id: 80 [2025-01-14 17:07:40.177] [ERROR] [grpc-default-executor-816] region CDC error: region: 139136532, resolvedTs:455303663539191810 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:07:40.177] [ERROR] [grpc-default-executor-816] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139136532 , resolvedTs : 455303663539191810 [2025-01-14 17:07:40.578] [INFO] [TidbEventSorter] lastHeartBeatTs : 455303108231168001 , lastSendHeartBeatTs : 1736844140243 , minResolvedTs : 1736843522 455303108231168001 , {139128648=455303668782071809, 139131720=455303631387754500, 139136584=455303668782071809, 139135816=455303301570756627, 139136588=455303293682581564, 137464374=455303617270775818, 139135820=455303519373099009, 139065664=455303577005457411, 139135808=455303334325649479, 139135812=455303355611742215, 139136580=455303307875319809, 139135832=455303351391223831, 139136600=455303652489822232, 139143768=455303668782071809, 134760989=455303668782071809, 138058799=455303622251773957, 134210281=455303477156380674, 139136604=455303614636228662, 139135836=455303583559057436, 139135824=455303362099806209, 139136592=455303566598340633, 139133520=455303300260298756, 137463084=455303628831326231, 139136596=455303658794385410, 139135828=455303535455633419, 131623161=455303275303927811, 139131752=455303455383748616, 139140200=455303597623083074, 139133800=455303419771486211, 139136616=455303271319339011, 131623165=455303330170667011, 139136620=455303345099767840, 131623409=455303561696247984, 133206489=455303308320964616, 139136608=455303336475230278, 138071575=455303582772625413, 139136612=455303282552209494, 130992353=455303362099806209, 139136632=455303659842961417, 130989561=455303350119825441, 133320645=455303562220535887, 139136636=455303477156380674, 130820597=455303668782071809, 137533705=455303477561131010, 139136624=455303286602334216, 134273597=455303292631646280, 137533711=455303668782071809, 139136628=455303508310622234, 139133812=455303668782071809, 131545749=455303668664107016, 131045517=455303615960317953, 139136520=455303545115639813, 139135752=455303407332753409, 129007605=455303276614647811, 137492087=455303562220535887, 139135756=455303423441502235, 139136524=455303356136030209, 130958217=455303560123384015, 133509565=455303562220535887, 139136512=455303647246942209, 139135744=455303359635652611, 131765393=455303362099806209, 139135748=455303407424503809, 137533309=455303668782071809, 139136516=455303272931524629, 138100330=455303358036574212, 132648121=455303477156380674, 139135768=455303423441502235, 138100334=455303271974699013, 139135772=455303455383748616, 127235020=455303668664107016, 127518659=455303477156380674, 139120144=455303310234615809, 133425833=455303324102295553, 139135760=455303423441502235, 139136528=455303515414724613, 137473898=455303614125572097, 138055782=455303559861502024, 138767896=455303518062379018, 139136532=455303663539191810, 139135764=455303423441502235, 131619513=455303336214659117, 130610857=455303561434104160, 138776612=455303668782071809, 138827044=455303511993745418, 138081885=455303284138180610, 139135788=455303333853790255, 131686577=455303444111556610, 139135776=455303315949355042, 127596788=455303303405764609, 139135780=455303455383748616, 126613476=455303348717355012, 132620181=455303561696247984, 137997899=455303430270353415, 132640149=455303288070864966, 139133240=455303668782071809, 139135800=455303455383748616, 132587665=455303668664107016, 139135804=455303519386206210, 137544777=455303533884604422, 130514245=455303477561131010, 139100104=455303425709047819, 139136712=455303477156380674, 131641437=455303623064158215, 139124940=455303279236087860, 139136716=455303668782071809, 131401309=455303668664107016, 135574429=455303477561131010, 129670449=455303668782071809, 139136704=455303361929412614, 139000004=455303650392670209, 139136708=455303425538916377, 137463720=455303668664107016, 139136720=455303529754001413, 136705209=455303604084932619, 131402865=455303571631767558, 131054953=455303668782071809, 137512084=455303275946180618, 139132896=455303303838564369, 131687285=455303536795713543, 138007444=455303668782071809, 132795205=455303295266193443, 139133412=455303668782071809, 131905645=455303668782071809, 130763641=455303287808196672, 139136504=455303537461035011, 139136508=455303361405124613, 131422573=455303668664107016, 130725745=455303362099806209, 139136496=455303647246942209, 131844197=455303430532497453, 139136500=455303427913940993, 137798644=455303668664107016, 139136648=455303320209195025, 126675016=455303477156380674, 139136652=455303562220535887, 138780556=455303561434104160, 138906508=455303668782071809, 139136640=455303477156380674, 130825217=455303333853790255, 139131780=455303615357124622, 131739669=455303668782071809, 139136644=455303589392285702, 137470945=455303271451197441, 139124632=455303583559057436, 137533921=455303324627107846, 130111765=455303652489822232, 126799444=455303362099806209, 139136656=455303279761686589, 127663168=455303362099806209, 139124628=455303618057207828, 133944357=455303108231168001, 132931593=455303540096106502, 139136684=455303477156380674, 131729469=455303477156380674, 131710773=455303477156380674, 139136696=455303668782071809, 137797824=455303308320964616, 131011641=455303561434104160, 139136700=455303358298718214, 139136688=455303534407057413, 139134128=455303279381315631, 139136692=455303355677278224} , [455303108231168001, 455303271319339011, 455303271451197441, 455303271974699013, 455303272931524629, 455303275303927811, 455303275946180618, 455303276614647811, 455303279236087860, 455303279381315631, 455303279761686589, 455303282552209494, 455303284138180610, 455303286602334216, 455303287808196672, 455303288070864966, 455303292631646280, 455303293682581564, 455303295266193443, 455303300260298756, 455303301570756627, 455303303405764609, 455303303838564369, 455303307875319809, 455303308320964616 x 2, 455303310234615809, 455303315949355042, 455303320209195025, 455303324102295553, 455303324627107846, 455303330170667011, 455303333853790255 x 2, 455303334325649479, 455303336214659117, 455303336475230278, 455303345099767840, 455303348717355012, 455303350119825441, 455303351391223831, 455303355611742215, 455303355677278224, 455303356136030209, 455303358036574212, 455303358298718214, 455303359635652611, 455303361405124613, 455303361929412614, 455303362099806209 x 6, 455303407332753409, 455303407424503809, 455303419771486211, 455303423441502235 x 4, 455303425538916377, 455303425709047819, 455303427913940993, 455303430270353415, 455303430532497453, 455303444111556610, 455303455383748616 x 4, 455303477156380674 x 10, 455303477561131010 x 3, 455303508310622234, 455303511993745418, 455303515414724613, 455303518062379018, 455303519373099009, 455303519386206210, 455303529754001413, 455303533884604422, 455303534407057413, 455303535455633419, 455303536795713543, 455303537461035011, 455303540096106502, 455303545115639813, 455303559861502024, 455303560123384015, 455303561434104160 x 3, 455303561696247984 x 2, 455303562220535887 x 4, 455303566598340633, 455303571631767558, 455303577005457411, 455303582772625413, 455303583559057436 x 2, 455303589392285702, 455303597623083074, 455303604084932619, 455303614125572097, 455303614636228662, 455303615357124622, 455303615960317953, 455303617270775818, 455303618057207828, 455303622251773957, 455303623064158215, 455303628831326231, 455303631387754500, 455303647246942209 x 2, 455303650392670209, 455303652489822232 x 2, 455303658794385410, 455303659842961417, 455303663539191810, 455303668664107016 x 7, 455303668782071809 x 19] [2025-01-14 17:07:40.716] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:41.329] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:41.856] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:41.878] [INFO] [grpc-default-executor-812] handle resolvedTs: 1736845661883, regionId: 132587665 [2025-01-14 17:07:41.878] [INFO] [grpc-default-executor-805] handle resolvedTs: 1736845661883, regionId: 137463720 [2025-01-14 17:07:41.878] [INFO] [grpc-default-executor-816] handle resolvedTs: 1736845661883, regionId: 127235020 [2025-01-14 17:07:41.878] [INFO] [grpc-default-executor-805] handle resolvedTs: 1736845661883, regionId: 131422573 [2025-01-14 17:07:41.878] [INFO] [grpc-default-executor-812] handle resolvedTs: 1736845661883, regionId: 137798644 [2025-01-14 17:07:41.878] [INFO] [grpc-default-executor-805] handle resolvedTs: 1736845661883, regionId: 131401309 [2025-01-14 17:07:41.878] [INFO] [grpc-default-executor-812] handle resolvedTs: 1736845661883, regionId: 131545749 [2025-01-14 17:07:42.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-816] handle resolvedTs: 1736845662432, regionId: 134760989 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-805] handle resolvedTs: 1736845662432, regionId: 129670449 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-812] handle resolvedTs: 1736845662432, regionId: 139133240 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-816] handle resolvedTs: 1736845662432, regionId: 137533711 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-804] handle resolvedTs: 1736845662432, regionId: 138007444 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-812] handle resolvedTs: 1736845662432, regionId: 139133812 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-802] handle resolvedTs: 1736845662432, regionId: 138776612 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-802] handle resolvedTs: 1736845662432, regionId: 139136584 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-804] handle resolvedTs: 1736845662432, regionId: 137533309 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-802] handle resolvedTs: 1736845662432, regionId: 138906508 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-804] handle resolvedTs: 1736845662432, regionId: 139133412 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-812] handle resolvedTs: 1736845662432, regionId: 130820597 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-812] handle resolvedTs: 1736845662432, regionId: 139143768 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-802] handle resolvedTs: 1736845662432, regionId: 131905645 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-812] handle resolvedTs: 1736845662432, regionId: 139136696 [2025-01-14 17:07:42.421] [INFO] [grpc-default-executor-816] handle resolvedTs: 1736845662432, regionId: 131739669 [2025-01-14 17:07:42.422] [INFO] [grpc-default-executor-812] handle resolvedTs: 1736845662432, regionId: 131054953 [2025-01-14 17:07:42.422] [INFO] [grpc-default-executor-816] handle resolvedTs: 1736845662432, regionId: 139128648 [2025-01-14 17:07:42.422] [INFO] [grpc-default-executor-802] handle resolvedTs: 1736845662432, regionId: 139136716 [2025-01-14 17:07:46.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:46.095] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:51.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:52.393] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:07:55.996] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:07:58.380] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:03.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:03.729] [ERROR] [grpc-default-executor-805] onErrorEventHandle : region_id: 138776612 error { not_leader { region_id: 138776612 } } request_id: 84 [2025-01-14 17:08:03.729] [ERROR] [grpc-default-executor-805] region CDC error: region: 138776612, resolvedTs:455303674837598215 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138776612 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:08:03.729] [ERROR] [grpc-default-executor-805] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138776612 } , regionId: 138776612 , resolvedTs : 455303674837598215 [2025-01-14 17:08:03.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:07.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:08.291] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:14.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:14.350] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:17.121] [ERROR] [grpc-default-executor-805] onErrorEventHandle : region_id: 134760989 error { not_leader { region_id: 134760989 } } request_id: 15 [2025-01-14 17:08:17.121] [ERROR] [grpc-default-executor-805] region CDC error: region: 134760989, resolvedTs:455303678245470270 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 134760989 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:08:17.121] [ERROR] [grpc-default-executor-805] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 134760989 } , regionId: 134760989 , resolvedTs : 455303678245470270 [2025-01-14 17:08:17.634] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:19.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:19.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:22.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:22.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:25.446] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:29.054] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:29.581] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:31.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:32.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:33.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:33.484] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:34.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:34.388] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:34.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:35.314] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:38.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:38.948] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:39.754] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:40.667] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:41.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:42.471] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:52.067] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:52.658] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:56.554] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:56.648] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:08:57.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:08:57.616] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:09:02.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:09:03.154] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:09:07.457] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:09:07.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:09:08.566] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:09:08.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:09:13.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:09:17.602] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:09:18.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:09:19.957] [INFO] [region_error_worker] remove regions: [138776612, 134760989, 139136532] [2025-01-14 17:09:19.958] [INFO] [region_error_worker] close (region: 138776612) [2025-01-14 17:09:19.958] [INFO] [region_error_worker] terminated (region: 138776612) [2025-01-14 17:09:19.958] [INFO] [region_error_worker] close (region: 134760989) [2025-01-14 17:09:19.958] [INFO] [region_error_worker] terminated (region: 134760989) [2025-01-14 17:09:19.958] [INFO] [region_error_worker] close (region: 139136532) [2025-01-14 17:09:19.958] [INFO] [region_error_worker] terminated (region: 139136532) [2025-01-14 17:09:19.958] [INFO] [region_error_worker] regions error and restart [138776612, 134760989, 139136532] [2025-01-14 17:09:24.446] [ERROR] [grpc-default-executor-802] onErrorEventHandle : region_id: 139136584 error { epoch_not_match { } } request_id: 4 [2025-01-14 17:09:24.446] [ERROR] [grpc-default-executor-802] region CDC error: region: 139136584, resolvedTs:455303694275575809 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:09:24.446] [ERROR] [grpc-default-executor-802] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139136584 , resolvedTs : 455303694275575809 [2025-01-14 17:09:25.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:09:31.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:09:38.006] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:09:40.350] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:09:40.350] [WARN] [region_error_worker] 43.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 44.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:09:40.350] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:09:41.554] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:09:44.930] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:09:45.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:09:45.464] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:09:49.148] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:09:49.260] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:09:50.030] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:09:50.627] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:00.254] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:00.463] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:04.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:04.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:09.226] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:09.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:11.611] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:12.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:16.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:17.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:23.554] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:24.222] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:24.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:25.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:26.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:26.909] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:32.354] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:32.448] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:33.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:33.768] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:44.009] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:44.591] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:45.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:45.165] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:46.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:46.370] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:51.454] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:51.510] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:52.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:52.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:55.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:56.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:10:56.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:10:57.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:02.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:02.602] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:03.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:03.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:10.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:11.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:11.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:12.249] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:13.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:13.354] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:17.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:17.888] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:18.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:18.468] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:19.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:19.957] [INFO] [region_error_worker] remove regions: [139136584] [2025-01-14 17:11:19.958] [INFO] [region_error_worker] close (region: 139136584) [2025-01-14 17:11:19.958] [INFO] [region_error_worker] terminated (region: 139136584) [2025-01-14 17:11:19.958] [INFO] [region_error_worker] regions error and restart [139136584] [2025-01-14 17:11:20.355] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:23.596] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:23.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:24.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:24.763] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:29.555] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:29.943] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:34.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:35.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:36.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:36.838] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:40.104] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:11:40.104] [WARN] [region_error_worker] 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 46.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 47.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:11:40.105] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:11:40.427] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:40.696] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:44.894] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:44.966] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:46.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:46.360] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:51.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:52.072] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:55.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:56.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:56.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:57.208] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:11:58.991] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:11:59.044] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:00.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:00.249] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:04.711] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:05.554] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:12.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:12.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:16.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:17.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:20.315] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:20.615] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:24.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:25.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:27.754] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:28.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:31.431] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:32.001] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:35.258] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:35.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:40.354] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:40.615] [INFO] [TidbEventSorter] lastHeartBeatTs : 455303108231168001 , lastSendHeartBeatTs : 1736844140243 , minResolvedTs : 1736843522 455303108231168001 , {139128648=455303747346628609, 139131720=455303631387754500, 139136584=455303694275575809, 139135816=455303301570756627, 139136588=455303293682581564, 137464374=455303617270775818, 139135820=455303519373099009, 139065664=455303577005457411, 139135808=455303334325649479, 139135812=455303355611742215, 139136580=455303307875319809, 139135832=455303351391223831, 139136600=455303652489822232, 139143768=455303747346628609, 134760989=455303678245470270, 138058799=455303622251773957, 134210281=455303477156380674, 139136604=455303614636228662, 139135836=455303583559057436, 139135824=455303362099806209, 139136592=455303566598340633, 139133520=455303300260298756, 137463084=455303628831326231, 139136596=455303658794385410, 139135828=455303535455633419, 131623161=455303275303927811, 139131752=455303455383748616, 139140200=455303597623083074, 139133800=455303419771486211, 139136616=455303271319339011, 131623165=455303330170667011, 139136620=455303345099767840, 131623409=455303561696247984, 133206489=455303308320964616, 139136608=455303336475230278, 138071575=455303582772625413, 139136612=455303282552209494, 130992353=455303362099806209, 139136632=455303659842961417, 130989561=455303350119825441, 133320645=455303562220535887, 139136636=455303477156380674, 130820597=455303747346628609, 137533705=455303477561131010, 139136624=455303286602334216, 134273597=455303292631646280, 137533711=455303747346628609, 139136628=455303508310622234, 139133812=455303747346628609, 131545749=455303747294199813, 131045517=455303615960317953, 139136520=455303545115639813, 139135752=455303407332753409, 129007605=455303276614647811, 137492087=455303562220535887, 139135756=455303423441502235, 139136524=455303356136030209, 130958217=455303560123384015, 133509565=455303562220535887, 139136512=455303647246942209, 139135744=455303359635652611, 131765393=455303362099806209, 139135748=455303407424503809, 137533309=455303747346628609, 139136516=455303272931524629, 138100330=455303358036574212, 132648121=455303477156380674, 139135768=455303423441502235, 138100334=455303271974699013, 139135772=455303455383748616, 127235020=455303747294199813, 127518659=455303477156380674, 139120144=455303310234615809, 133425833=455303324102295553, 139135760=455303423441502235, 139136528=455303515414724613, 137473898=455303614125572097, 138055782=455303559861502024, 138767896=455303518062379018, 139136532=455303663539191810, 139135764=455303423441502235, 131619513=455303336214659117, 130610857=455303561434104160, 138776612=455303674837598215, 138827044=455303511993745418, 138081885=455303284138180610, 139135788=455303333853790255, 131686577=455303444111556610, 139135776=455303315949355042, 127596788=455303303405764609, 139135780=455303455383748616, 126613476=455303348717355012, 132620181=455303561696247984, 137997899=455303430270353415, 132640149=455303288070864966, 139133240=455303747346628609, 139135800=455303455383748616, 132587665=455303747294199813, 139135804=455303519386206210, 137544777=455303533884604422, 130514245=455303477561131010, 139100104=455303425709047819, 139136712=455303477156380674, 131641437=455303623064158215, 139124940=455303279236087860, 139136716=455303747346628609, 131401309=455303747294199813, 135574429=455303477561131010, 129670449=455303747346628609, 139136704=455303361929412614, 139000004=455303650392670209, 139136708=455303425538916377, 137463720=455303747294199813, 139136720=455303529754001413, 136705209=455303604084932619, 131402865=455303571631767558, 131054953=455303747346628609, 137512084=455303275946180618, 139132896=455303303838564369, 131687285=455303536795713543, 138007444=455303747346628609, 132795205=455303295266193443, 139133412=455303747346628609, 131905645=455303747346628609, 130763641=455303287808196672, 139136504=455303537461035011, 139136508=455303361405124613, 131422573=455303747294199813, 130725745=455303362099806209, 139136496=455303647246942209, 131844197=455303430532497453, 139136500=455303427913940993, 137798644=455303747294199813, 139136648=455303320209195025, 126675016=455303477156380674, 139136652=455303562220535887, 138780556=455303561434104160, 138906508=455303747346628609, 139136640=455303477156380674, 130825217=455303333853790255, 139131780=455303615357124622, 131739669=455303747346628609, 139136644=455303589392285702, 137470945=455303271451197441, 139124632=455303583559057436, 137533921=455303324627107846, 130111765=455303652489822232, 126799444=455303362099806209, 139136656=455303279761686589, 127663168=455303362099806209, 139124628=455303618057207828, 133944357=455303108231168001, 132931593=455303540096106502, 139136684=455303477156380674, 131729469=455303477156380674, 131710773=455303477156380674, 139136696=455303747346628609, 137797824=455303308320964616, 131011641=455303561434104160, 139136700=455303358298718214, 139136688=455303534407057413, 139134128=455303279381315631, 139136692=455303355677278224} , [455303108231168001, 455303271319339011, 455303271451197441, 455303271974699013, 455303272931524629, 455303275303927811, 455303275946180618, 455303276614647811, 455303279236087860, 455303279381315631, 455303279761686589, 455303282552209494, 455303284138180610, 455303286602334216, 455303287808196672, 455303288070864966, 455303292631646280, 455303293682581564, 455303295266193443, 455303300260298756, 455303301570756627, 455303303405764609, 455303303838564369, 455303307875319809, 455303308320964616 x 2, 455303310234615809, 455303315949355042, 455303320209195025, 455303324102295553, 455303324627107846, 455303330170667011, 455303333853790255 x 2, 455303334325649479, 455303336214659117, 455303336475230278, 455303345099767840, 455303348717355012, 455303350119825441, 455303351391223831, 455303355611742215, 455303355677278224, 455303356136030209, 455303358036574212, 455303358298718214, 455303359635652611, 455303361405124613, 455303361929412614, 455303362099806209 x 6, 455303407332753409, 455303407424503809, 455303419771486211, 455303423441502235 x 4, 455303425538916377, 455303425709047819, 455303427913940993, 455303430270353415, 455303430532497453, 455303444111556610, 455303455383748616 x 4, 455303477156380674 x 10, 455303477561131010 x 3, 455303508310622234, 455303511993745418, 455303515414724613, 455303518062379018, 455303519373099009, 455303519386206210, 455303529754001413, 455303533884604422, 455303534407057413, 455303535455633419, 455303536795713543, 455303537461035011, 455303540096106502, 455303545115639813, 455303559861502024, 455303560123384015, 455303561434104160 x 3, 455303561696247984 x 2, 455303562220535887 x 4, 455303566598340633, 455303571631767558, 455303577005457411, 455303582772625413, 455303583559057436 x 2, 455303589392285702, 455303597623083074, 455303604084932619, 455303614125572097, 455303614636228662, 455303615357124622, 455303615960317953, 455303617270775818, 455303618057207828, 455303622251773957, 455303623064158215, 455303628831326231, 455303631387754500, 455303647246942209 x 2, 455303650392670209, 455303652489822232 x 2, 455303658794385410, 455303659842961417, 455303663539191810, 455303674837598215, 455303678245470270, 455303694275575809, 455303747294199813 x 7, 455303747346628609 x 16] [2025-01-14 17:12:40.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:41.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:41.633] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:41.921] [INFO] [grpc-default-executor-849] handle resolvedTs: 1736845961932, regionId: 132587665 [2025-01-14 17:12:41.921] [INFO] [grpc-default-executor-827] handle resolvedTs: 1736845961932, regionId: 137463720 [2025-01-14 17:12:41.921] [INFO] [grpc-default-executor-834] handle resolvedTs: 1736845961932, regionId: 131422573 [2025-01-14 17:12:41.921] [INFO] [grpc-default-executor-827] handle resolvedTs: 1736845961932, regionId: 137798644 [2025-01-14 17:12:41.921] [INFO] [grpc-default-executor-854] handle resolvedTs: 1736845961932, regionId: 127235020 [2025-01-14 17:12:41.921] [INFO] [grpc-default-executor-834] handle resolvedTs: 1736845961932, regionId: 131401309 [2025-01-14 17:12:41.921] [INFO] [grpc-default-executor-834] handle resolvedTs: 1736845961932, regionId: 131545749 [2025-01-14 17:12:42.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:43.069] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:43.109] [INFO] [grpc-default-executor-854] handle resolvedTs: 1736845963034, regionId: 139133240 [2025-01-14 17:12:43.109] [INFO] [grpc-default-executor-834] handle resolvedTs: 1736845963034, regionId: 129670449 [2025-01-14 17:12:43.109] [INFO] [grpc-default-executor-854] handle resolvedTs: 1736845963034, regionId: 137533711 [2025-01-14 17:12:43.109] [INFO] [grpc-default-executor-854] handle resolvedTs: 1736845963034, regionId: 138007444 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-827] handle resolvedTs: 1736845963034, regionId: 139133812 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-851] handle resolvedTs: 1736845963034, regionId: 137533309 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-854] handle resolvedTs: 1736845963034, regionId: 139133412 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-834] handle resolvedTs: 1736845963034, regionId: 138906508 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-834] handle resolvedTs: 1736845963034, regionId: 130820597 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-854] handle resolvedTs: 1736845963034, regionId: 131905645 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-827] handle resolvedTs: 1736845963034, regionId: 139143768 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-827] handle resolvedTs: 1736845963034, regionId: 139136696 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-834] handle resolvedTs: 1736845963034, regionId: 131739669 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-827] handle resolvedTs: 1736845963034, regionId: 139128648 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-849] handle resolvedTs: 1736845963034, regionId: 139136716 [2025-01-14 17:12:43.110] [INFO] [grpc-default-executor-854] handle resolvedTs: 1736845963034, regionId: 131054953 [2025-01-14 17:12:50.260] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:50.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:52.748] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:53.360] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:53.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:53.892] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:56.710] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:12:56.773] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:12:59.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:00.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:01.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:02.044] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:04.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:07.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:07.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:07.968] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:12.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:12.825] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:13.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:15.455] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:21.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:21.201] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:22.099] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:25.120] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:30.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:30.770] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:40.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:43.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:47.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:48.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:53.907] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:53.995] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:54.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:55.721] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:56.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:56.314] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:13:56.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:13:57.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:14:01.403] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:14:03.464] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:14:07.686] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:14:07.954] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:14:08.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:14:08.580] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:14:13.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:14:13.667] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:14:31.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:14:32.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:14:36.454] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:14:37.053] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:14:40.653] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:14:40.989] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:14:41.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:14:42.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:14:50.645] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:14:51.206] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:14:59.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:00.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:08.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:09.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:16.739] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:17.287] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:19.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:19.372] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:20.554] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:21.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:22.354] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:26.624] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:27.153] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:33.258] [ERROR] [grpc-default-executor-849] onErrorEventHandle : region_id: 139133240 error { epoch_not_match { } } request_id: 96 [2025-01-14 17:15:33.259] [ERROR] [grpc-default-executor-849] region CDC error: region: 139133240, resolvedTs:455303791006711810 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:15:33.259] [ERROR] [grpc-default-executor-849] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139133240 , resolvedTs : 455303791006711810 [2025-01-14 17:15:33.529] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:43.353] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:43.953] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:44.553] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:44.893] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:45.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:46.694] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:50.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:50.285] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:52.656] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:52.959] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:15:57.453] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:15:57.836] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:16:05.253] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:16:05.853] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:16:06.753] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:16:42.159] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:16:43.096] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:16:43.101] [WARN] [pool-6-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:16:44.559] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:16:44.859] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:16:55.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:16:55.959] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:17:00.859] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:17:01.065] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:17:01.659] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:17:02.316] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:17:12.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:17:13.059] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:17:19.957] [INFO] [region_error_worker] remove regions: [139133240] [2025-01-14 17:17:19.957] [INFO] [region_error_worker] close (region: 139133240) [2025-01-14 17:17:19.957] [INFO] [region_error_worker] terminated (region: 139133240) [2025-01-14 17:17:19.958] [INFO] [region_error_worker] regions error and restart [139133240] [2025-01-14 17:17:23.260] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:17:23.342] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:17:38.259] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:17:38.561] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:17:40.011] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:17:40.011] [WARN] [region_error_worker] 44.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 46.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:17:40.012] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:17:40.013] [ERROR] [grpc-default-executor-890] onErrorEventHandle : region_id: 139136696 error { epoch_not_match { } } request_id: 157 [2025-01-14 17:17:40.013] [ERROR] [grpc-default-executor-890] region CDC error: region: 139136696, resolvedTs:455303824954359810 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:17:40.013] [ERROR] [grpc-default-executor-890] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139136696 , resolvedTs : 455303824954359810 [2025-01-14 17:17:40.664] [INFO] [TidbEventSorter] lastHeartBeatTs : 455303108231168001 , lastSendHeartBeatTs : 1736844140243 , minResolvedTs : 1736843522 455303108231168001 , {139128648=455303826002935811, 139131720=455303631387754500, 139136584=455303694275575809, 139135816=455303301570756627, 139136588=455303293682581564, 137464374=455303617270775818, 139135820=455303519373099009, 139065664=455303577005457411, 139135808=455303334325649479, 139135812=455303355611742215, 139136580=455303307875319809, 139135832=455303351391223831, 139136600=455303652489822232, 139143768=455303826002935811, 134760989=455303678245470270, 138058799=455303622251773957, 134210281=455303477156380674, 139136604=455303614636228662, 139135836=455303583559057436, 139135824=455303362099806209, 139136592=455303566598340633, 139133520=455303300260298756, 137463084=455303628831326231, 139136596=455303658794385410, 139135828=455303535455633419, 131623161=455303275303927811, 139131752=455303455383748616, 139140200=455303597623083074, 139133800=455303419771486211, 139136616=455303271319339011, 131623165=455303330170667011, 139136620=455303345099767840, 131623409=455303561696247984, 133206489=455303308320964616, 139136608=455303336475230278, 138071575=455303582772625413, 139136612=455303282552209494, 130992353=455303362099806209, 139136632=455303659842961417, 130989561=455303350119825441, 133320645=455303562220535887, 139136636=455303477156380674, 130820597=455303826002935811, 137533705=455303477561131010, 139136624=455303286602334216, 134273597=455303292631646280, 137533711=455303826002935811, 139136628=455303508310622234, 139133812=455303826002935811, 131545749=455303825963614216, 131045517=455303615960317953, 139136520=455303545115639813, 139135752=455303407332753409, 129007605=455303276614647811, 137492087=455303562220535887, 139135756=455303423441502235, 139136524=455303356136030209, 130958217=455303560123384015, 133509565=455303562220535887, 139136512=455303647246942209, 139135744=455303359635652611, 131765393=455303362099806209, 139135748=455303407424503809, 137533309=455303826002935811, 139136516=455303272931524629, 138100330=455303358036574212, 132648121=455303477156380674, 139135768=455303423441502235, 138100334=455303271974699013, 139135772=455303455383748616, 127235020=455303825963614216, 127518659=455303477156380674, 139120144=455303310234615809, 133425833=455303324102295553, 139135760=455303423441502235, 139136528=455303515414724613, 137473898=455303614125572097, 138055782=455303559861502024, 138767896=455303518062379018, 139136532=455303663539191810, 139135764=455303423441502235, 131619513=455303336214659117, 130610857=455303561434104160, 138776612=455303674837598215, 138827044=455303511993745418, 138081885=455303284138180610, 139135788=455303333853790255, 131686577=455303444111556610, 139135776=455303315949355042, 127596788=455303303405764609, 139135780=455303455383748616, 126613476=455303348717355012, 132620181=455303561696247984, 137997899=455303430270353415, 132640149=455303288070864966, 139133240=455303791006711810, 139135800=455303455383748616, 132587665=455303825963614216, 139135804=455303519386206210, 137544777=455303533884604422, 130514245=455303477561131010, 139100104=455303425709047819, 139136712=455303477156380674, 131641437=455303623064158215, 139124940=455303279236087860, 139136716=455303826002935811, 131401309=455303825963614216, 135574429=455303477561131010, 129670449=455303826002935811, 139136704=455303361929412614, 139000004=455303650392670209, 139136708=455303425538916377, 137463720=455303825963614216, 139136720=455303529754001413, 136705209=455303604084932619, 131402865=455303571631767558, 131054953=455303826002935811, 137512084=455303275946180618, 139132896=455303303838564369, 131687285=455303536795713543, 138007444=455303826002935811, 132795205=455303295266193443, 139133412=455303826002935811, 131905645=455303826002935811, 130763641=455303287808196672, 139136504=455303537461035011, 139136508=455303361405124613, 131422573=455303825963614216, 130725745=455303362099806209, 139136496=455303647246942209, 131844197=455303430532497453, 139136500=455303427913940993, 137798644=455303825963614216, 139136648=455303320209195025, 126675016=455303477156380674, 139136652=455303562220535887, 138780556=455303561434104160, 138906508=455303826002935811, 139136640=455303477156380674, 130825217=455303333853790255, 139131780=455303615357124622, 131739669=455303826002935811, 139136644=455303589392285702, 137470945=455303271451197441, 139124632=455303583559057436, 137533921=455303324627107846, 130111765=455303652489822232, 126799444=455303362099806209, 139136656=455303279761686589, 127663168=455303362099806209, 139124628=455303618057207828, 133944357=455303108231168001, 132931593=455303540096106502, 139136684=455303477156380674, 131729469=455303477156380674, 131710773=455303477156380674, 139136696=455303824954359810, 137797824=455303308320964616, 131011641=455303561434104160, 139136700=455303358298718214, 139136688=455303534407057413, 139134128=455303279381315631, 139136692=455303355677278224} , [455303108231168001, 455303271319339011, 455303271451197441, 455303271974699013, 455303272931524629, 455303275303927811, 455303275946180618, 455303276614647811, 455303279236087860, 455303279381315631, 455303279761686589, 455303282552209494, 455303284138180610, 455303286602334216, 455303287808196672, 455303288070864966, 455303292631646280, 455303293682581564, 455303295266193443, 455303300260298756, 455303301570756627, 455303303405764609, 455303303838564369, 455303307875319809, 455303308320964616 x 2, 455303310234615809, 455303315949355042, 455303320209195025, 455303324102295553, 455303324627107846, 455303330170667011, 455303333853790255 x 2, 455303334325649479, 455303336214659117, 455303336475230278, 455303345099767840, 455303348717355012, 455303350119825441, 455303351391223831, 455303355611742215, 455303355677278224, 455303356136030209, 455303358036574212, 455303358298718214, 455303359635652611, 455303361405124613, 455303361929412614, 455303362099806209 x 6, 455303407332753409, 455303407424503809, 455303419771486211, 455303423441502235 x 4, 455303425538916377, 455303425709047819, 455303427913940993, 455303430270353415, 455303430532497453, 455303444111556610, 455303455383748616 x 4, 455303477156380674 x 10, 455303477561131010 x 3, 455303508310622234, 455303511993745418, 455303515414724613, 455303518062379018, 455303519373099009, 455303519386206210, 455303529754001413, 455303533884604422, 455303534407057413, 455303535455633419, 455303536795713543, 455303537461035011, 455303540096106502, 455303545115639813, 455303559861502024, 455303560123384015, 455303561434104160 x 3, 455303561696247984 x 2, 455303562220535887 x 4, 455303566598340633, 455303571631767558, 455303577005457411, 455303582772625413, 455303583559057436 x 2, 455303589392285702, 455303597623083074, 455303604084932619, 455303614125572097, 455303614636228662, 455303615357124622, 455303615960317953, 455303617270775818, 455303618057207828, 455303622251773957, 455303623064158215, 455303628831326231, 455303631387754500, 455303647246942209 x 2, 455303650392670209, 455303652489822232 x 2, 455303658794385410, 455303659842961417, 455303663539191810, 455303674837598215, 455303678245470270, 455303694275575809, 455303791006711810, 455303824954359810, 455303825963614216 x 7, 455303826002935811 x 14] [2025-01-14 17:17:41.930] [INFO] [grpc-default-executor-857] handle resolvedTs: 1736846261932, regionId: 132587665 [2025-01-14 17:17:41.930] [INFO] [grpc-default-executor-886] handle resolvedTs: 1736846261932, regionId: 137463720 [2025-01-14 17:17:41.930] [INFO] [grpc-default-executor-888] handle resolvedTs: 1736846261932, regionId: 127235020 [2025-01-14 17:17:41.930] [INFO] [grpc-default-executor-874] handle resolvedTs: 1736846261932, regionId: 131422573 [2025-01-14 17:17:41.930] [INFO] [grpc-default-executor-881] handle resolvedTs: 1736846261932, regionId: 137798644 [2025-01-14 17:17:41.930] [INFO] [grpc-default-executor-881] handle resolvedTs: 1736846261932, regionId: 131401309 [2025-01-14 17:17:41.931] [INFO] [grpc-default-executor-888] handle resolvedTs: 1736846261932, regionId: 131545749 [2025-01-14 17:17:43.360] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:17:43.400] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-886] handle resolvedTs: 1736846264082, regionId: 129670449 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-874] handle resolvedTs: 1736846264082, regionId: 137533711 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-857] handle resolvedTs: 1736846264082, regionId: 138007444 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-857] handle resolvedTs: 1736846264082, regionId: 139133812 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-886] handle resolvedTs: 1736846264082, regionId: 137533309 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-886] handle resolvedTs: 1736846264082, regionId: 138906508 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-886] handle resolvedTs: 1736846264082, regionId: 139133412 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-886] handle resolvedTs: 1736846264082, regionId: 130820597 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-886] handle resolvedTs: 1736846264082, regionId: 131905645 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-874] handle resolvedTs: 1736846264082, regionId: 139143768 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-857] handle resolvedTs: 1736846264082, regionId: 131739669 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-881] handle resolvedTs: 1736846264082, regionId: 131054953 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-857] handle resolvedTs: 1736846264082, regionId: 139128648 [2025-01-14 17:17:44.303] [INFO] [grpc-default-executor-857] handle resolvedTs: 1736846264082, regionId: 139136716 [2025-01-14 17:17:48.159] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:17:49.420] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:17:58.391] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:17:58.441] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:17:59.259] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:00.189] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:04.659] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:04.736] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:13.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:13.736] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:14.559] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:14.577] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:19.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:19.437] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:22.758] [ERROR] [grpc-default-executor-874] onErrorEventHandle : region_id: 138906508 error { not_leader { region_id: 138906508 } } request_id: 138 [2025-01-14 17:18:22.758] [ERROR] [grpc-default-executor-874] region CDC error: region: 138906508, resolvedTs:455303837027401733 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138906508 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:18:22.758] [ERROR] [grpc-default-executor-874] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138906508 } , regionId: 138906508 , resolvedTs : 455303837027401733 [2025-01-14 17:18:26.960] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:27.517] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:31.059] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:31.118] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:31.659] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:32.036] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:36.812] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:37.081] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:40.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:42.172] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:42.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:46.059] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:48.159] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:48.459] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:52.959] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:52.990] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:54.207] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:54.769] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:56.859] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:57.809] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:18:59.259] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:18:59.333] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:19:06.559] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:19:06.794] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:19:14.261] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:19:14.591] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:19:15.459] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:19:16.059] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:19:19.957] [INFO] [region_error_worker] remove regions: [139136696, 138906508] [2025-01-14 17:19:19.957] [INFO] [region_error_worker] close (region: 139136696) [2025-01-14 17:19:19.957] [INFO] [region_error_worker] terminated (region: 139136696) [2025-01-14 17:19:19.957] [INFO] [region_error_worker] close (region: 138906508) [2025-01-14 17:19:19.957] [INFO] [region_error_worker] terminated (region: 138906508) [2025-01-14 17:19:19.958] [INFO] [region_error_worker] regions error and restart [139136696, 138906508] [2025-01-14 17:19:20.559] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:19:20.625] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:19:21.159] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:19:21.459] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:19:35.023] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:19:35.117] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:19:40.080] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:19:40.080] [WARN] [region_error_worker] 43.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 44.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:19:40.081] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:19:40.082] [ERROR] [grpc-default-executor-881] onErrorEventHandle : region_id: 139136716 error { epoch_not_match { } } request_id: 106 [2025-01-14 17:19:40.083] [ERROR] [grpc-default-executor-881] region CDC error: region: 139136716, resolvedTs:455303857040785445 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:19:40.083] [ERROR] [grpc-default-executor-881] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139136716 , resolvedTs : 455303857040785445 [2025-01-14 17:19:40.959] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:19:40.980] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:19:58.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:19:58.669] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:19:59.260] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:19:59.559] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:07.457] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:07.469] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:08.861] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:08.905] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:12.459] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:12.555] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:13.128] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:14.259] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:21.859] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:22.358] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:23.996] [ERROR] [grpc-default-executor-895] onErrorEventHandle : region_id: 139128648 error { not_leader { region_id: 139128648 } } request_id: 1 [2025-01-14 17:20:23.996] [ERROR] [grpc-default-executor-895] region CDC error: region: 139128648, resolvedTs:455303868876587075 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139128648 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:20:23.997] [ERROR] [grpc-default-executor-895] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139128648 } , regionId: 139128648 , resolvedTs : 455303868876587075 [2025-01-14 17:20:24.402] [ERROR] [grpc-default-executor-901] onErrorEventHandle : region_id: 131739669 error { not_leader { region_id: 131739669 } } request_id: 142 [2025-01-14 17:20:24.402] [ERROR] [grpc-default-executor-901] region CDC error: region: 131739669, resolvedTs:455303868876587075 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131739669 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:20:24.402] [ERROR] [grpc-default-executor-901] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131739669 } , regionId: 131739669 , resolvedTs : 455303868876587075 [2025-01-14 17:20:26.367] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:26.374] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:33.159] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:33.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:37.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:37.664] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:40.162] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:40.539] [ERROR] [grpc-default-executor-895] onErrorEventHandle : region_id: 139133412 error { epoch_not_match { } } request_id: 123 [2025-01-14 17:20:40.539] [ERROR] [grpc-default-executor-895] region CDC error: region: 139133412, resolvedTs:455303873087143937 java.lang.RuntimeException: regionCDC error:epoch_not_match { } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:20:40.539] [ERROR] [grpc-default-executor-895] handle error: java.lang.RuntimeException: regionCDC error:epoch_not_match { } , regionId: 139133412 , resolvedTs : 455303873087143937 [2025-01-14 17:20:40.959] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:44.649] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:44.725] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:48.459] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:48.503] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:51.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:52.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:54.481] [ERROR] [grpc-default-executor-896] onErrorEventHandle : region_id: 137533309 error { not_leader { region_id: 137533309 } } request_id: 64 [2025-01-14 17:20:54.481] [ERROR] [grpc-default-executor-896] region CDC error: region: 137533309, resolvedTs:455303876754276379 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 137533309 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:20:54.481] [ERROR] [grpc-default-executor-896] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 137533309 } , regionId: 137533309 , resolvedTs : 455303876754276379 [2025-01-14 17:20:55.659] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:55.672] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:56.259] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:56.559] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:20:57.459] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:20:59.914] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:00.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:02.259] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:08.159] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:08.161] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:10.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:10.986] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:13.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:14.342] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:15.560] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:16.113] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:17.859] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:18.474] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:19.957] [INFO] [region_error_worker] remove regions: [139128648, 139136716, 131739669, 139133412, 137533309] [2025-01-14 17:21:19.957] [INFO] [region_error_worker] close (region: 139128648) [2025-01-14 17:21:19.957] [INFO] [region_error_worker] terminated (region: 139128648) [2025-01-14 17:21:19.958] [INFO] [region_error_worker] close (region: 139136716) [2025-01-14 17:21:19.958] [INFO] [region_error_worker] terminated (region: 139136716) [2025-01-14 17:21:19.958] [INFO] [region_error_worker] close (region: 131739669) [2025-01-14 17:21:19.958] [INFO] [region_error_worker] terminated (region: 131739669) [2025-01-14 17:21:19.958] [INFO] [region_error_worker] close (region: 139133412) [2025-01-14 17:21:19.958] [INFO] [region_error_worker] terminated (region: 139133412) [2025-01-14 17:21:19.958] [INFO] [region_error_worker] close (region: 137533309) [2025-01-14 17:21:19.958] [INFO] [region_error_worker] terminated (region: 137533309) [2025-01-14 17:21:19.958] [INFO] [region_error_worker] regions error and restart [139128648, 139136716, 131739669, 139133412, 137533309] [2025-01-14 17:21:20.559] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:21.159] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:22.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:24.176] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:24.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:24.815] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:26.259] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:27.459] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:31.558] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:33.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:34.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:35.938] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:36.459] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:36.550] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:40.022] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:21:40.022] [WARN] [region_error_worker] 43.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 44.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:21:40.023] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:21:40.059] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:42.763] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:43.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:43.398] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:43.959] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:44.885] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:48.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:49.659] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:50.259] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:50.265] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:50.859] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:51.159] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:21:56.261] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:21:58.059] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:22:03.159] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:22:12.804] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:22:17.021] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:22:17.080] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:22:18.159] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:22:18.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:22:19.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:22:19.740] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:22:24.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:22:26.023] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:22:32.299] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:22:40.688] [INFO] [TidbEventSorter] lastHeartBeatTs : 455303108231168001 , lastSendHeartBeatTs : 1736844140243 , minResolvedTs : 1736843522 455303108231168001 , {139128648=455303868876587075, 139131720=455303631387754500, 139136584=455303694275575809, 139135816=455303301570756627, 139136588=455303293682581564, 137464374=455303617270775818, 139135820=455303519373099009, 139065664=455303577005457411, 139135808=455303334325649479, 139135812=455303355611742215, 139136580=455303307875319809, 139135832=455303351391223831, 139136600=455303652489822232, 139143768=455303904633028610, 134760989=455303678245470270, 138058799=455303622251773957, 134210281=455303477156380674, 139136604=455303614636228662, 139135836=455303583559057436, 139135824=455303362099806209, 139136592=455303566598340633, 139133520=455303300260298756, 137463084=455303628831326231, 139136596=455303658794385410, 139135828=455303535455633419, 131623161=455303275303927811, 139131752=455303455383748616, 139140200=455303597623083074, 139133800=455303419771486211, 139136616=455303271319339011, 131623165=455303330170667011, 139136620=455303345099767840, 131623409=455303561696247984, 133206489=455303308320964616, 139136608=455303336475230278, 138071575=455303582772625413, 139136612=455303282552209494, 130992353=455303362099806209, 139136632=455303659842961417, 130989561=455303350119825441, 133320645=455303562220535887, 139136636=455303477156380674, 130820597=455303904633028610, 137533705=455303477561131010, 139136624=455303286602334216, 134273597=455303292631646280, 137533711=455303904633028610, 139136628=455303508310622234, 139133812=455303904633028610, 131545749=455303904554909701, 131045517=455303615960317953, 139136520=455303545115639813, 139135752=455303407332753409, 129007605=455303276614647811, 137492087=455303562220535887, 139135756=455303423441502235, 139136524=455303356136030209, 130958217=455303560123384015, 133509565=455303562220535887, 139136512=455303647246942209, 139135744=455303359635652611, 131765393=455303362099806209, 139135748=455303407424503809, 137533309=455303876754276379, 139136516=455303272931524629, 138100330=455303358036574212, 132648121=455303477156380674, 139135768=455303423441502235, 138100334=455303271974699013, 139135772=455303455383748616, 127235020=455303904554909701, 127518659=455303477156380674, 139120144=455303310234615809, 133425833=455303324102295553, 139135760=455303423441502235, 139136528=455303515414724613, 137473898=455303614125572097, 138055782=455303559861502024, 138767896=455303518062379018, 139136532=455303663539191810, 139135764=455303423441502235, 131619513=455303336214659117, 130610857=455303561434104160, 138776612=455303674837598215, 138827044=455303511993745418, 138081885=455303284138180610, 139135788=455303333853790255, 131686577=455303444111556610, 139135776=455303315949355042, 127596788=455303303405764609, 139135780=455303455383748616, 126613476=455303348717355012, 132620181=455303561696247984, 137997899=455303430270353415, 132640149=455303288070864966, 139133240=455303791006711810, 139135800=455303455383748616, 132587665=455303904554909701, 139135804=455303519386206210, 137544777=455303533884604422, 130514245=455303477561131010, 139100104=455303425709047819, 139136712=455303477156380674, 131641437=455303623064158215, 139124940=455303279236087860, 139136716=455303857040785445, 131401309=455303904554909701, 135574429=455303477561131010, 129670449=455303904633028610, 139136704=455303361929412614, 139000004=455303650392670209, 139136708=455303425538916377, 137463720=455303904554909701, 139136720=455303529754001413, 136705209=455303604084932619, 131402865=455303571631767558, 131054953=455303904633028610, 137512084=455303275946180618, 139132896=455303303838564369, 131687285=455303536795713543, 138007444=455303904633028610, 132795205=455303295266193443, 139133412=455303873087143937, 131905645=455303904633028610, 130763641=455303287808196672, 139136504=455303537461035011, 139136508=455303361405124613, 131422573=455303904554909701, 130725745=455303362099806209, 139136496=455303647246942209, 131844197=455303430532497453, 139136500=455303427913940993, 137798644=455303904554909701, 139136648=455303320209195025, 126675016=455303477156380674, 139136652=455303562220535887, 138780556=455303561434104160, 138906508=455303837027401733, 139136640=455303477156380674, 130825217=455303333853790255, 139131780=455303615357124622, 131739669=455303868876587075, 139136644=455303589392285702, 137470945=455303271451197441, 139124632=455303583559057436, 137533921=455303324627107846, 130111765=455303652489822232, 126799444=455303362099806209, 139136656=455303279761686589, 127663168=455303362099806209, 139124628=455303618057207828, 133944357=455303108231168001, 132931593=455303540096106502, 139136684=455303477156380674, 131729469=455303477156380674, 131710773=455303477156380674, 139136696=455303824954359810, 137797824=455303308320964616, 131011641=455303561434104160, 139136700=455303358298718214, 139136688=455303534407057413, 139134128=455303279381315631, 139136692=455303355677278224} , [455303108231168001, 455303271319339011, 455303271451197441, 455303271974699013, 455303272931524629, 455303275303927811, 455303275946180618, 455303276614647811, 455303279236087860, 455303279381315631, 455303279761686589, 455303282552209494, 455303284138180610, 455303286602334216, 455303287808196672, 455303288070864966, 455303292631646280, 455303293682581564, 455303295266193443, 455303300260298756, 455303301570756627, 455303303405764609, 455303303838564369, 455303307875319809, 455303308320964616 x 2, 455303310234615809, 455303315949355042, 455303320209195025, 455303324102295553, 455303324627107846, 455303330170667011, 455303333853790255 x 2, 455303334325649479, 455303336214659117, 455303336475230278, 455303345099767840, 455303348717355012, 455303350119825441, 455303351391223831, 455303355611742215, 455303355677278224, 455303356136030209, 455303358036574212, 455303358298718214, 455303359635652611, 455303361405124613, 455303361929412614, 455303362099806209 x 6, 455303407332753409, 455303407424503809, 455303419771486211, 455303423441502235 x 4, 455303425538916377, 455303425709047819, 455303427913940993, 455303430270353415, 455303430532497453, 455303444111556610, 455303455383748616 x 4, 455303477156380674 x 10, 455303477561131010 x 3, 455303508310622234, 455303511993745418, 455303515414724613, 455303518062379018, 455303519373099009, 455303519386206210, 455303529754001413, 455303533884604422, 455303534407057413, 455303535455633419, 455303536795713543, 455303537461035011, 455303540096106502, 455303545115639813, 455303559861502024, 455303560123384015, 455303561434104160 x 3, 455303561696247984 x 2, 455303562220535887 x 4, 455303566598340633, 455303571631767558, 455303577005457411, 455303582772625413, 455303583559057436 x 2, 455303589392285702, 455303597623083074, 455303604084932619, 455303614125572097, 455303614636228662, 455303615357124622, 455303615960317953, 455303617270775818, 455303618057207828, 455303622251773957, 455303623064158215, 455303628831326231, 455303631387754500, 455303647246942209 x 2, 455303650392670209, 455303652489822232 x 2, 455303658794385410, 455303659842961417, 455303663539191810, 455303674837598215, 455303678245470270, 455303694275575809, 455303791006711810, 455303824954359810, 455303837027401733, 455303857040785445, 455303868876587075 x 2, 455303873087143937, 455303876754276379, 455303904554909701 x 7, 455303904633028610 x 8] [2025-01-14 17:22:42.758] [INFO] [grpc-default-executor-925] handle resolvedTs: 1736846562732, regionId: 132587665 [2025-01-14 17:22:42.758] [INFO] [grpc-default-executor-918] handle resolvedTs: 1736846562732, regionId: 137463720 [2025-01-14 17:22:42.758] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846562732, regionId: 127235020 [2025-01-14 17:22:42.758] [INFO] [grpc-default-executor-919] handle resolvedTs: 1736846562732, regionId: 131422573 [2025-01-14 17:22:42.758] [INFO] [grpc-default-executor-918] handle resolvedTs: 1736846562732, regionId: 137798644 [2025-01-14 17:22:42.758] [INFO] [grpc-default-executor-918] handle resolvedTs: 1736846562732, regionId: 131401309 [2025-01-14 17:22:42.758] [INFO] [grpc-default-executor-919] handle resolvedTs: 1736846562732, regionId: 131545749 [2025-01-14 17:22:45.032] [INFO] [grpc-default-executor-918] handle resolvedTs: 1736846565032, regionId: 129670449 [2025-01-14 17:22:45.032] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846565032, regionId: 137533711 [2025-01-14 17:22:45.032] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846565032, regionId: 138007444 [2025-01-14 17:22:45.032] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846565032, regionId: 139133812 [2025-01-14 17:22:45.032] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846565032, regionId: 130820597 [2025-01-14 17:22:45.032] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846565032, regionId: 139143768 [2025-01-14 17:22:45.032] [INFO] [grpc-default-executor-916] handle resolvedTs: 1736846565032, regionId: 131905645 [2025-01-14 17:22:45.033] [INFO] [grpc-default-executor-925] handle resolvedTs: 1736846565032, regionId: 131054953 [2025-01-14 17:22:46.658] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:22:50.925] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:24:06.759] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:24:38.050] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:24:49.403] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:24:49.959] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:24:50.559] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:24:54.460] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:24:58.359] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:24:59.559] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:25:37.730] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:25:41.259] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] is not reachable [2025-01-14 17:26:56.559] [WARN] [pool-10-thread-1] store [10.178.1.72:26620] recovers to be reachable [2025-01-14 17:27:01.921] [ERROR] [grpc-default-executor-916] onErrorEventHandle : region_id: 139133812 error { not_leader { region_id: 139133812 } } request_id: 49 [2025-01-14 17:27:01.922] [ERROR] [grpc-default-executor-916] region CDC error: region: 139133812, resolvedTs:455303973223006216 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139133812 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:27:01.922] [ERROR] [grpc-default-executor-916] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139133812 } , regionId: 139133812 , resolvedTs : 455303973223006216 [2025-01-14 17:27:19.957] [INFO] [region_error_worker] remove regions: [139133812] [2025-01-14 17:27:19.957] [INFO] [region_error_worker] close (region: 139133812) [2025-01-14 17:27:19.957] [INFO] [region_error_worker] terminated (region: 139133812) [2025-01-14 17:27:19.957] [INFO] [region_error_worker] regions error and restart [139133812] [2025-01-14 17:27:40.164] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:27:40.165] [WARN] [region_error_worker] 42.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 43.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 44.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:27:40.165] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:27:40.716] [INFO] [TidbEventSorter] lastHeartBeatTs : 455303108231168001 , lastSendHeartBeatTs : 1736844140243 , minResolvedTs : 1736843522 455303108231168001 , {139128648=455303868876587075, 139131720=455303631387754500, 139136584=455303694275575809, 139135816=455303301570756627, 139136588=455303293682581564, 137464374=455303617270775818, 139135820=455303519373099009, 139065664=455303577005457411, 139135808=455303334325649479, 139135812=455303355611742215, 139136580=455303307875319809, 139135832=455303351391223831, 139136600=455303652489822232, 139143768=455303983341764652, 134760989=455303678245470270, 138058799=455303622251773957, 134210281=455303477156380674, 139136604=455303614636228662, 139135836=455303583559057436, 139135824=455303362099806209, 139136592=455303566598340633, 139133520=455303300260298756, 137463084=455303628831326231, 139136596=455303658794385410, 139135828=455303535455633419, 131623161=455303275303927811, 139131752=455303455383748616, 139140200=455303597623083074, 139133800=455303419771486211, 139136616=455303271319339011, 131623165=455303330170667011, 139136620=455303345099767840, 131623409=455303561696247984, 133206489=455303308320964616, 139136608=455303336475230278, 138071575=455303582772625413, 139136612=455303282552209494, 130992353=455303362099806209, 139136632=455303659842961417, 130989561=455303350119825441, 133320645=455303562220535887, 139136636=455303477156380674, 130820597=455303983341764652, 137533705=455303477561131010, 139136624=455303286602334216, 134273597=455303292631646280, 137533711=455303983341764652, 139136628=455303508310622234, 139133812=455303973223006216, 131545749=455303983354871833, 131045517=455303615960317953, 139136520=455303545115639813, 139135752=455303407332753409, 129007605=455303276614647811, 137492087=455303562220535887, 139135756=455303423441502235, 139136524=455303356136030209, 130958217=455303560123384015, 133509565=455303562220535887, 139136512=455303647246942209, 139135744=455303359635652611, 131765393=455303362099806209, 139135748=455303407424503809, 137533309=455303876754276379, 139136516=455303272931524629, 138100330=455303358036574212, 132648121=455303477156380674, 139135768=455303423441502235, 138100334=455303271974699013, 139135772=455303455383748616, 127235020=455303983354871833, 127518659=455303477156380674, 139120144=455303310234615809, 133425833=455303324102295553, 139135760=455303423441502235, 139136528=455303515414724613, 137473898=455303614125572097, 138055782=455303559861502024, 138767896=455303518062379018, 139136532=455303663539191810, 139135764=455303423441502235, 131619513=455303336214659117, 130610857=455303561434104160, 138776612=455303674837598215, 138827044=455303511993745418, 138081885=455303284138180610, 139135788=455303333853790255, 131686577=455303444111556610, 139135776=455303315949355042, 127596788=455303303405764609, 139135780=455303455383748616, 126613476=455303348717355012, 132620181=455303561696247984, 137997899=455303430270353415, 132640149=455303288070864966, 139133240=455303791006711810, 139135800=455303455383748616, 132587665=455303983354871833, 139135804=455303519386206210, 137544777=455303533884604422, 130514245=455303477561131010, 139100104=455303425709047819, 139136712=455303477156380674, 131641437=455303623064158215, 139124940=455303279236087860, 139136716=455303857040785445, 131401309=455303983354871833, 135574429=455303477561131010, 129670449=455303983341764652, 139136704=455303361929412614, 139000004=455303650392670209, 139136708=455303425538916377, 137463720=455303983354871833, 139136720=455303529754001413, 136705209=455303604084932619, 131402865=455303571631767558, 131054953=455303983341764652, 137512084=455303275946180618, 139132896=455303303838564369, 131687285=455303536795713543, 138007444=455303983341764652, 132795205=455303295266193443, 139133412=455303873087143937, 131905645=455303983341764652, 130763641=455303287808196672, 139136504=455303537461035011, 139136508=455303361405124613, 131422573=455303983354871833, 130725745=455303362099806209, 139136496=455303647246942209, 131844197=455303430532497453, 139136500=455303427913940993, 137798644=455303983354871833, 139136648=455303320209195025, 126675016=455303477156380674, 139136652=455303562220535887, 138780556=455303561434104160, 138906508=455303837027401733, 139136640=455303477156380674, 130825217=455303333853790255, 139131780=455303615357124622, 131739669=455303868876587075, 139136644=455303589392285702, 137470945=455303271451197441, 139124632=455303583559057436, 137533921=455303324627107846, 130111765=455303652489822232, 126799444=455303362099806209, 139136656=455303279761686589, 127663168=455303362099806209, 139124628=455303618057207828, 133944357=455303108231168001, 132931593=455303540096106502, 139136684=455303477156380674, 131729469=455303477156380674, 131710773=455303477156380674, 139136696=455303824954359810, 137797824=455303308320964616, 131011641=455303561434104160, 139136700=455303358298718214, 139136688=455303534407057413, 139134128=455303279381315631, 139136692=455303355677278224} , [455303108231168001, 455303271319339011, 455303271451197441, 455303271974699013, 455303272931524629, 455303275303927811, 455303275946180618, 455303276614647811, 455303279236087860, 455303279381315631, 455303279761686589, 455303282552209494, 455303284138180610, 455303286602334216, 455303287808196672, 455303288070864966, 455303292631646280, 455303293682581564, 455303295266193443, 455303300260298756, 455303301570756627, 455303303405764609, 455303303838564369, 455303307875319809, 455303308320964616 x 2, 455303310234615809, 455303315949355042, 455303320209195025, 455303324102295553, 455303324627107846, 455303330170667011, 455303333853790255 x 2, 455303334325649479, 455303336214659117, 455303336475230278, 455303345099767840, 455303348717355012, 455303350119825441, 455303351391223831, 455303355611742215, 455303355677278224, 455303356136030209, 455303358036574212, 455303358298718214, 455303359635652611, 455303361405124613, 455303361929412614, 455303362099806209 x 6, 455303407332753409, 455303407424503809, 455303419771486211, 455303423441502235 x 4, 455303425538916377, 455303425709047819, 455303427913940993, 455303430270353415, 455303430532497453, 455303444111556610, 455303455383748616 x 4, 455303477156380674 x 10, 455303477561131010 x 3, 455303508310622234, 455303511993745418, 455303515414724613, 455303518062379018, 455303519373099009, 455303519386206210, 455303529754001413, 455303533884604422, 455303534407057413, 455303535455633419, 455303536795713543, 455303537461035011, 455303540096106502, 455303545115639813, 455303559861502024, 455303560123384015, 455303561434104160 x 3, 455303561696247984 x 2, 455303562220535887 x 4, 455303566598340633, 455303571631767558, 455303577005457411, 455303582772625413, 455303583559057436 x 2, 455303589392285702, 455303597623083074, 455303604084932619, 455303614125572097, 455303614636228662, 455303615357124622, 455303615960317953, 455303617270775818, 455303618057207828, 455303622251773957, 455303623064158215, 455303628831326231, 455303631387754500, 455303647246942209 x 2, 455303650392670209, 455303652489822232 x 2, 455303658794385410, 455303659842961417, 455303663539191810, 455303674837598215, 455303678245470270, 455303694275575809, 455303791006711810, 455303824954359810, 455303837027401733, 455303857040785445, 455303868876587075 x 2, 455303873087143937, 455303876754276379, 455303973223006216, 455303983341764652 x 7, 455303983354871833 x 7] [2025-01-14 17:27:43.363] [INFO] [grpc-default-executor-932] handle resolvedTs: 1736846863332, regionId: 132587665 [2025-01-14 17:27:43.363] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846863332, regionId: 127235020 [2025-01-14 17:27:43.363] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846863332, regionId: 137463720 [2025-01-14 17:27:43.363] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846863332, regionId: 131422573 [2025-01-14 17:27:43.363] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846863332, regionId: 137798644 [2025-01-14 17:27:43.363] [INFO] [grpc-default-executor-932] handle resolvedTs: 1736846863332, regionId: 131401309 [2025-01-14 17:27:43.363] [INFO] [grpc-default-executor-930] handle resolvedTs: 1736846863332, regionId: 131545749 [2025-01-14 17:27:45.707] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846865682, regionId: 129670449 [2025-01-14 17:27:45.707] [INFO] [grpc-default-executor-932] handle resolvedTs: 1736846865682, regionId: 137533711 [2025-01-14 17:27:45.707] [INFO] [grpc-default-executor-932] handle resolvedTs: 1736846865682, regionId: 138007444 [2025-01-14 17:27:45.707] [INFO] [grpc-default-executor-915] handle resolvedTs: 1736846865682, regionId: 130820597 [2025-01-14 17:27:45.707] [INFO] [grpc-default-executor-932] handle resolvedTs: 1736846865682, regionId: 139143768 [2025-01-14 17:27:45.707] [INFO] [grpc-default-executor-930] handle resolvedTs: 1736846865682, regionId: 131905645 [2025-01-14 17:27:45.708] [INFO] [grpc-default-executor-930] handle resolvedTs: 1736846865682, regionId: 131054953 [2025-01-14 17:27:52.362] [ERROR] [grpc-default-executor-930] onErrorEventHandle : region_id: 131054953 error { not_leader { region_id: 131054953 } } request_id: 117 [2025-01-14 17:27:52.362] [ERROR] [grpc-default-executor-930] region CDC error: region: 131054953, resolvedTs:455303986343313411 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131054953 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:27:52.362] [ERROR] [grpc-default-executor-930] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131054953 } , regionId: 131054953 , resolvedTs : 455303986343313411 [2025-01-14 17:29:09.805] [ERROR] [grpc-default-executor-934] onErrorEventHandle : region_id: 139143768 error { not_leader { region_id: 139143768 } } request_id: 14 [2025-01-14 17:29:09.805] [ERROR] [grpc-default-executor-934] region CDC error: region: 139143768, resolvedTs:455304006646366212 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139143768 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:29:09.805] [ERROR] [grpc-default-executor-934] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 139143768 } , regionId: 139143768 , resolvedTs : 455304006646366212 [2025-01-14 17:29:19.957] [INFO] [region_error_worker] remove regions: [139143768, 131054953] [2025-01-14 17:29:19.958] [INFO] [region_error_worker] close (region: 139143768) [2025-01-14 17:29:19.958] [INFO] [region_error_worker] terminated (region: 139143768) [2025-01-14 17:29:19.958] [INFO] [region_error_worker] close (region: 131054953) [2025-01-14 17:29:19.958] [INFO] [region_error_worker] terminated (region: 131054953) [2025-01-14 17:29:19.958] [INFO] [region_error_worker] regions error and restart [139143768, 131054953] [2025-01-14 17:29:40.218] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:29:40.218] [WARN] [region_error_worker] 44.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 46.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:29:40.218] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:31:02.768] [ERROR] [grpc-default-executor-941] onErrorEventHandle : region_id: 130820597 error { not_leader { region_id: 130820597 } } request_id: 43 [2025-01-14 17:31:02.768] [ERROR] [grpc-default-executor-941] region CDC error: region: 130820597, resolvedTs:455304036321067020 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 130820597 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:31:02.768] [ERROR] [grpc-default-executor-941] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 130820597 } , regionId: 130820597 , resolvedTs : 455304036321067020 [2025-01-14 17:31:13.771] [ERROR] [grpc-default-executor-942] onErrorEventHandle : region_id: 138007444 error { not_leader { region_id: 138007444 } } request_id: 121 [2025-01-14 17:31:13.771] [ERROR] [grpc-default-executor-942] region CDC error: region: 138007444, resolvedTs:455304039217758210 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138007444 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:31:13.771] [ERROR] [grpc-default-executor-942] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 138007444 } , regionId: 138007444 , resolvedTs : 455304039217758210 [2025-01-14 17:31:19.957] [INFO] [region_error_worker] remove regions: [130820597, 138007444] [2025-01-14 17:31:19.957] [INFO] [region_error_worker] close (region: 130820597) [2025-01-14 17:31:19.957] [INFO] [region_error_worker] terminated (region: 130820597) [2025-01-14 17:31:19.958] [INFO] [region_error_worker] close (region: 138007444) [2025-01-14 17:31:19.958] [INFO] [region_error_worker] terminated (region: 138007444) [2025-01-14 17:31:19.958] [INFO] [region_error_worker] regions error and restart [130820597, 138007444] [2025-01-14 17:31:40.339] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:31:40.339] [WARN] [region_error_worker] 44.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 46.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:31:40.339] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:32:04.752] [ERROR] [grpc-default-executor-945] onErrorEventHandle : region_id: 129670449 error { not_leader { region_id: 129670449 } } request_id: 109 [2025-01-14 17:32:04.753] [ERROR] [grpc-default-executor-945] region CDC error: region: 129670449, resolvedTs:455304052128350210 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 129670449 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:32:04.753] [ERROR] [grpc-default-executor-945] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 129670449 } , regionId: 129670449 , resolvedTs : 455304052128350210 [2025-01-14 17:32:21.114] [ERROR] [grpc-default-executor-934] onErrorEventHandle : region_id: 131422573 error { not_leader { region_id: 131422573 } } request_id: 128 [2025-01-14 17:32:21.114] [ERROR] [grpc-default-executor-934] region CDC error: region: 131422573, resolvedTs:455304056781406213 java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131422573 } at org.tikv.cdc.RegionCDCClient.onErrorEventHandle(RegionCDCClient.java:182) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:148) at org.tikv.cdc.RegionCDCClient.onNext(RegionCDCClient.java:49) at org.tikv.shade.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:493) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:691) at org.tikv.shade.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:675) at org.tikv.shade.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at org.tikv.shade.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) [2025-01-14 17:32:21.114] [ERROR] [grpc-default-executor-934] handle error: java.lang.RuntimeException: regionCDC error:not_leader { region_id: 131422573 } , regionId: 131422573 , resolvedTs : 455304056781406213 [2025-01-14 17:32:40.752] [INFO] [TidbEventSorter] lastHeartBeatTs : 455303108231168001 , lastSendHeartBeatTs : 1736844140243 , minResolvedTs : 1736843522 455303108231168001 , {139128648=455303868876587075, 139131720=455303631387754500, 139136584=455303694275575809, 139135816=455303301570756627, 139136588=455303293682581564, 137464374=455303617270775818, 139135820=455303519373099009, 139065664=455303577005457411, 139135808=455303334325649479, 139135812=455303355611742215, 139136580=455303307875319809, 139135832=455303351391223831, 139136600=455303652489822232, 139143768=455304006646366212, 134760989=455303678245470270, 138058799=455303622251773957, 134210281=455303477156380674, 139136604=455303614636228662, 139135836=455303583559057436, 139135824=455303362099806209, 139136592=455303566598340633, 139133520=455303300260298756, 137463084=455303628831326231, 139136596=455303658794385410, 139135828=455303535455633419, 131623161=455303275303927811, 139131752=455303455383748616, 139140200=455303597623083074, 139133800=455303419771486211, 139136616=455303271319339011, 131623165=455303330170667011, 139136620=455303345099767840, 131623409=455303561696247984, 133206489=455303308320964616, 139136608=455303336475230278, 138071575=455303582772625413, 139136612=455303282552209494, 130992353=455303362099806209, 139136632=455303659842961417, 130989561=455303350119825441, 133320645=455303562220535887, 139136636=455303477156380674, 130820597=455304036321067020, 137533705=455303477561131010, 139136624=455303286602334216, 134273597=455303292631646280, 137533711=455304061578903572, 139136628=455303508310622234, 139133812=455303973223006216, 131545749=455304062037393410, 131045517=455303615960317953, 139136520=455303545115639813, 139135752=455303407332753409, 129007605=455303276614647811, 137492087=455303562220535887, 139135756=455303423441502235, 139136524=455303356136030209, 130958217=455303560123384015, 133509565=455303562220535887, 139136512=455303647246942209, 139135744=455303359635652611, 131765393=455303362099806209, 139135748=455303407424503809, 137533309=455303876754276379, 139136516=455303272931524629, 138100330=455303358036574212, 132648121=455303477156380674, 139135768=455303423441502235, 138100334=455303271974699013, 139135772=455303455383748616, 127235020=455304062037393410, 127518659=455303477156380674, 139120144=455303310234615809, 133425833=455303324102295553, 139135760=455303423441502235, 139136528=455303515414724613, 137473898=455303614125572097, 138055782=455303559861502024, 138767896=455303518062379018, 139136532=455303663539191810, 139135764=455303423441502235, 131619513=455303336214659117, 130610857=455303561434104160, 138776612=455303674837598215, 138827044=455303511993745418, 138081885=455303284138180610, 139135788=455303333853790255, 131686577=455303444111556610, 139135776=455303315949355042, 127596788=455303303405764609, 139135780=455303455383748616, 126613476=455303348717355012, 132620181=455303561696247984, 137997899=455303430270353415, 132640149=455303288070864966, 139133240=455303791006711810, 139135800=455303455383748616, 132587665=455304062037393410, 139135804=455303519386206210, 137544777=455303533884604422, 130514245=455303477561131010, 139100104=455303425709047819, 139136712=455303477156380674, 131641437=455303623064158215, 139124940=455303279236087860, 139136716=455303857040785445, 131401309=455304062037393410, 135574429=455303477561131010, 129670449=455304052128350210, 139136704=455303361929412614, 139000004=455303650392670209, 139136708=455303425538916377, 137463720=455304062037393410, 139136720=455303529754001413, 136705209=455303604084932619, 131402865=455303571631767558, 131054953=455303986343313411, 137512084=455303275946180618, 139132896=455303303838564369, 131687285=455303536795713543, 138007444=455304039217758210, 132795205=455303295266193443, 139133412=455303873087143937, 131905645=455304061578903572, 130763641=455303287808196672, 139136504=455303537461035011, 139136508=455303361405124613, 131422573=455304056781406213, 130725745=455303362099806209, 139136496=455303647246942209, 131844197=455303430532497453, 139136500=455303427913940993, 137798644=455304062037393410, 139136648=455303320209195025, 126675016=455303477156380674, 139136652=455303562220535887, 138780556=455303561434104160, 138906508=455303837027401733, 139136640=455303477156380674, 130825217=455303333853790255, 139131780=455303615357124622, 131739669=455303868876587075, 139136644=455303589392285702, 137470945=455303271451197441, 139124632=455303583559057436, 137533921=455303324627107846, 130111765=455303652489822232, 126799444=455303362099806209, 139136656=455303279761686589, 127663168=455303362099806209, 139124628=455303618057207828, 133944357=455303108231168001, 132931593=455303540096106502, 139136684=455303477156380674, 131729469=455303477156380674, 131710773=455303477156380674, 139136696=455303824954359810, 137797824=455303308320964616, 131011641=455303561434104160, 139136700=455303358298718214, 139136688=455303534407057413, 139134128=455303279381315631, 139136692=455303355677278224} , [455303108231168001, 455303271319339011, 455303271451197441, 455303271974699013, 455303272931524629, 455303275303927811, 455303275946180618, 455303276614647811, 455303279236087860, 455303279381315631, 455303279761686589, 455303282552209494, 455303284138180610, 455303286602334216, 455303287808196672, 455303288070864966, 455303292631646280, 455303293682581564, 455303295266193443, 455303300260298756, 455303301570756627, 455303303405764609, 455303303838564369, 455303307875319809, 455303308320964616 x 2, 455303310234615809, 455303315949355042, 455303320209195025, 455303324102295553, 455303324627107846, 455303330170667011, 455303333853790255 x 2, 455303334325649479, 455303336214659117, 455303336475230278, 455303345099767840, 455303348717355012, 455303350119825441, 455303351391223831, 455303355611742215, 455303355677278224, 455303356136030209, 455303358036574212, 455303358298718214, 455303359635652611, 455303361405124613, 455303361929412614, 455303362099806209 x 6, 455303407332753409, 455303407424503809, 455303419771486211, 455303423441502235 x 4, 455303425538916377, 455303425709047819, 455303427913940993, 455303430270353415, 455303430532497453, 455303444111556610, 455303455383748616 x 4, 455303477156380674 x 10, 455303477561131010 x 3, 455303508310622234, 455303511993745418, 455303515414724613, 455303518062379018, 455303519373099009, 455303519386206210, 455303529754001413, 455303533884604422, 455303534407057413, 455303535455633419, 455303536795713543, 455303537461035011, 455303540096106502, 455303545115639813, 455303559861502024, 455303560123384015, 455303561434104160 x 3, 455303561696247984 x 2, 455303562220535887 x 4, 455303566598340633, 455303571631767558, 455303577005457411, 455303582772625413, 455303583559057436 x 2, 455303589392285702, 455303597623083074, 455303604084932619, 455303614125572097, 455303614636228662, 455303615357124622, 455303615960317953, 455303617270775818, 455303618057207828, 455303622251773957, 455303623064158215, 455303628831326231, 455303631387754500, 455303647246942209 x 2, 455303650392670209, 455303652489822232 x 2, 455303658794385410, 455303659842961417, 455303663539191810, 455303674837598215, 455303678245470270, 455303694275575809, 455303791006711810, 455303824954359810, 455303837027401733, 455303857040785445, 455303868876587075 x 2, 455303873087143937, 455303876754276379, 455303973223006216, 455303986343313411, 455304006646366212, 455304036321067020, 455304039217758210, 455304052128350210, 455304056781406213, 455304061578903572 x 2, 455304062037393410 x 6] [2025-01-14 17:32:43.472] [INFO] [grpc-default-executor-941] handle resolvedTs: 1736847163482, regionId: 132587665 [2025-01-14 17:32:43.472] [INFO] [grpc-default-executor-945] handle resolvedTs: 1736847163482, regionId: 127235020 [2025-01-14 17:32:43.472] [INFO] [grpc-default-executor-941] handle resolvedTs: 1736847163482, regionId: 137463720 [2025-01-14 17:32:43.472] [INFO] [grpc-default-executor-956] handle resolvedTs: 1736847163482, regionId: 137798644 [2025-01-14 17:32:43.472] [INFO] [grpc-default-executor-934] handle resolvedTs: 1736847163482, regionId: 131401309 [2025-01-14 17:32:43.472] [INFO] [grpc-default-executor-956] handle resolvedTs: 1736847163482, regionId: 131545749 [2025-01-14 17:32:45.763] [INFO] [grpc-default-executor-956] handle resolvedTs: 1736847165782, regionId: 137533711 [2025-01-14 17:32:45.764] [INFO] [grpc-default-executor-934] handle resolvedTs: 1736847165782, regionId: 131905645 [2025-01-14 17:33:19.957] [INFO] [region_error_worker] remove regions: [131422573, 129670449] [2025-01-14 17:33:19.957] [INFO] [region_error_worker] close (region: 131422573) [2025-01-14 17:33:19.957] [INFO] [region_error_worker] terminated (region: 131422573) [2025-01-14 17:33:19.958] [INFO] [region_error_worker] close (region: 129670449) [2025-01-14 17:33:19.958] [INFO] [region_error_worker] terminated (region: 129670449) [2025-01-14 17:33:19.958] [INFO] [region_error_worker] regions error and restart [131422573, 129670449] [2025-01-14 17:33:40.440] [WARN] [region_error_worker] BackOffer.maxSleep 20000ms is exceeded, errors: [2025-01-14 17:33:40.440] [WARN] [region_error_worker] 44.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 45.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" 46.org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" [2025-01-14 17:33:40.441] [ERROR] [region_error_worker] doRegionErrorEvent error org.tikv.common.exception.GrpcException: retry is exhausted. at org.tikv.common.util.ConcreteBackOffer.logThrowError(ConcreteBackOffer.java:235) at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:220) at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:160) at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:66) at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:108) at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:89) at org.tikv.common.PDClient.getRegionByID(PDClient.java:343) at org.tikv.common.region.RegionManager.getRegionById(RegionManager.java:157) at org.tikv.cdc.CDCClient.refreshStart(CDCClient.java:189) at org.tikv.cdc.CDCClient.doRegionErrorEvent(CDCClient.java:367) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:853) Caused by: org.tikv.common.exception.GrpcException: ErrorType: REGION_PEER_NOT_ELECTED Error: message: "Region Peer not elected. Please try later" at org.tikv.common.operation.PDErrorHandler.handleResponseError(PDErrorHandler.java:67) ... 13 common frames omitted [2025-01-14 17:37:40.782] [INFO] [TidbEventSorter] lastHeartBeatTs : 455303108231168001 , lastSendHeartBeatTs : 1736844140243 , minResolvedTs : 1736843522 455303108231168001 , {139128648=455303868876587075, 139131720=455303631387754500, 139136584=455303694275575809, 139135816=455303301570756627, 139136588=455303293682581564, 137464374=455303617270775818, 139135820=455303519373099009, 139065664=455303577005457411, 139135808=455303334325649479, 139135812=455303355611742215, 139136580=455303307875319809, 139135832=455303351391223831, 139136600=455303652489822232, 139143768=455304006646366212, 134760989=455303678245470270, 138058799=455303622251773957, 134210281=455303477156380674, 139136604=455303614636228662, 139135836=455303583559057436, 139135824=455303362099806209, 139136592=455303566598340633, 139133520=455303300260298756, 137463084=455303628831326231, 139136596=455303658794385410, 139135828=455303535455633419, 131623161=455303275303927811, 139131752=455303455383748616, 139140200=455303597623083074, 139133800=455303419771486211, 139136616=455303271319339011, 131623165=455303330170667011, 139136620=455303345099767840, 131623409=455303561696247984, 133206489=455303308320964616, 139136608=455303336475230278, 138071575=455303582772625413, 139136612=455303282552209494, 130992353=455303362099806209, 139136632=455303659842961417, 130989561=455303350119825441, 133320645=455303562220535887, 139136636=455303477156380674, 130820597=455304036321067020, 137533705=455303477561131010, 139136624=455303286602334216, 134273597=455303292631646280, 137533711=455304140707069954, 139136628=455303508310622234, 139133812=455303973223006216, 131545749=455304140707069953, 131045517=455303615960317953, 139136520=455303545115639813, 139135752=455303407332753409, 129007605=455303276614647811, 137492087=455303562220535887, 139135756=455303423441502235, 139136524=455303356136030209, 130958217=455303560123384015, 133509565=455303562220535887, 139136512=455303647246942209, 139135744=455303359635652611, 131765393=455303362099806209, 139135748=455303407424503809, 137533309=455303876754276379, 139136516=455303272931524629, 138100330=455303358036574212, 132648121=455303477156380674, 139135768=455303423441502235, 138100334=455303271974699013, 139135772=455303455383748616, 127235020=455304140707069953, 127518659=455303477156380674, 139120144=455303310234615809, 133425833=455303324102295553, 139135760=455303423441502235, 139136528=455303515414724613, 137473898=455303614125572097, 138055782=455303559861502024, 138767896=455303518062379018, 139136532=455303663539191810, 139135764=455303423441502235, 131619513=455303336214659117, 130610857=455303561434104160, 138776612=455303674837598215, 138827044=455303511993745418, 138081885=455303284138180610, 139135788=455303333853790255, 131686577=455303444111556610, 139135776=455303315949355042, 127596788=455303303405764609, 139135780=455303455383748616, 126613476=455303348717355012, 132620181=455303561696247984, 137997899=455303430270353415, 132640149=455303288070864966, 139133240=455303791006711810, 139135800=455303455383748616, 132587665=455304140707069953, 139135804=455303519386206210, 137544777=455303533884604422, 130514245=455303477561131010, 139100104=455303425709047819, 139136712=455303477156380674, 131641437=455303623064158215, 139124940=455303279236087860, 139136716=455303857040785445, 131401309=455304140707069953, 135574429=455303477561131010, 129670449=455304052128350210, 139136704=455303361929412614, 139000004=455303650392670209, 139136708=455303425538916377, 137463720=455304140707069953, 139136720=455303529754001413, 136705209=455303604084932619, 131402865=455303571631767558, 131054953=455303986343313411, 137512084=455303275946180618, 139132896=455303303838564369, 131687285=455303536795713543, 138007444=455304039217758210, 132795205=455303295266193443, 139133412=455303873087143937, 131905645=455304140707069954, 130763641=455303287808196672, 139136504=455303537461035011, 139136508=455303361405124613, 131422573=455304056781406213, 130725745=455303362099806209, 139136496=455303647246942209, 131844197=455303430532497453, 139136500=455303427913940993, 137798644=455304140707069953, 139136648=455303320209195025, 126675016=455303477156380674, 139136652=455303562220535887, 138780556=455303561434104160, 138906508=455303837027401733, 139136640=455303477156380674, 130825217=455303333853790255, 139131780=455303615357124622, 131739669=455303868876587075, 139136644=455303589392285702, 137470945=455303271451197441, 139124632=455303583559057436, 137533921=455303324627107846, 130111765=455303652489822232, 126799444=455303362099806209, 139136656=455303279761686589, 127663168=455303362099806209, 139124628=455303618057207828, 133944357=455303108231168001, 132931593=455303540096106502, 139136684=455303477156380674, 131729469=455303477156380674, 131710773=455303477156380674, 139136696=455303824954359810, 137797824=455303308320964616, 131011641=455303561434104160, 139136700=455303358298718214, 139136688=455303534407057413, 139134128=455303279381315631, 139136692=455303355677278224} , [455303108231168001, 455303271319339011, 455303271451197441, 455303271974699013, 455303272931524629, 455303275303927811, 455303275946180618, 455303276614647811, 455303279236087860, 455303279381315631, 455303279761686589, 455303282552209494, 455303284138180610, 455303286602334216, 455303287808196672, 455303288070864966, 455303292631646280, 455303293682581564, 455303295266193443, 455303300260298756, 455303301570756627, 455303303405764609, 455303303838564369, 455303307875319809, 455303308320964616 x 2, 455303310234615809, 455303315949355042, 455303320209195025, 455303324102295553, 455303324627107846, 455303330170667011, 455303333853790255 x 2, 455303334325649479, 455303336214659117, 455303336475230278, 455303345099767840, 455303348717355012, 455303350119825441, 455303351391223831, 455303355611742215, 455303355677278224, 455303356136030209, 455303358036574212, 455303358298718214, 455303359635652611, 455303361405124613, 455303361929412614, 455303362099806209 x 6, 455303407332753409, 455303407424503809, 455303419771486211, 455303423441502235 x 4, 455303425538916377, 455303425709047819, 455303427913940993, 455303430270353415, 455303430532497453, 455303444111556610, 455303455383748616 x 4, 455303477156380674 x 10, 455303477561131010 x 3, 455303508310622234, 455303511993745418, 455303515414724613, 455303518062379018, 455303519373099009, 455303519386206210, 455303529754001413, 455303533884604422, 455303534407057413, 455303535455633419, 455303536795713543, 455303537461035011, 455303540096106502, 455303545115639813, 455303559861502024, 455303560123384015, 455303561434104160 x 3, 455303561696247984 x 2, 455303562220535887 x 4, 455303566598340633, 455303571631767558, 455303577005457411, 455303582772625413, 455303583559057436 x 2, 455303589392285702, 455303597623083074, 455303604084932619, 455303614125572097, 455303614636228662, 455303615357124622, 455303615960317953, 455303617270775818, 455303618057207828, 455303622251773957, 455303623064158215, 455303628831326231, 455303631387754500, 455303647246942209 x 2, 455303650392670209, 455303652489822232 x 2, 455303658794385410, 455303659842961417, 455303663539191810, 455303674837598215, 455303678245470270, 455303694275575809, 455303791006711810, 455303824954359810, 455303837027401733, 455303857040785445, 455303868876587075 x 2, 455303873087143937, 455303876754276379, 455303973223006216, 455303986343313411, 455304006646366212, 455304036321067020, 455304039217758210, 455304052128350210, 455304056781406213, 455304140707069953 x 6, 455304140707069954 x 2] [2025-01-14 17:37:43.577] [INFO] [grpc-default-executor-956] handle resolvedTs: 1736847463582, regionId: 132587665 [2025-01-14 17:37:43.577] [INFO] [grpc-default-executor-948] handle resolvedTs: 1736847463582, regionId: 137463720 [2025-01-14 17:37:43.577] [INFO] [grpc-default-executor-962] handle resolvedTs: 1736847463582, regionId: 127235020 [2025-01-14 17:37:43.577] [INFO] [grpc-default-executor-956] handle resolvedTs: 1736847463582, regionId: 137798644 [2025-01-14 17:37:43.577] [INFO] [grpc-default-executor-970] handle resolvedTs: 1736847463582, regionId: 131401309 [2025-01-14 17:37:43.577] [INFO] [grpc-default-executor-962] handle resolvedTs: 1736847463582, regionId: 131545749 [2025-01-14 17:37:46.609] [INFO] [grpc-default-executor-956] handle resolvedTs: 1736847466593, regionId: 137533711 [2025-01-14 17:37:46.609] [INFO] [grpc-default-executor-962] handle resolvedTs: 1736847466593, regionId: 131905645