{"level":"INFO","ts":"2026-04-02T15:28:44+08:00","msg":"Modify obzone replica","controller":"obcluster","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBCluster","OBCluster":{"name":"obcluster","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster","reconcileID":"f1dbc770-e2e7-460b-b1ef-d4b2db8c62af","obzone":"zone1"}
{"level":"INFO","ts":"2026-04-02T15:28:44+08:00","msg":"Modify obzone replica","controller":"obcluster","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBCluster","OBCluster":{"name":"obcluster","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster","reconcileID":"f1dbc770-e2e7-460b-b1ef-d4b2db8c62af","obzone":"zone2"}
{"level":"INFO","ts":"2026-04-02T15:28:44+08:00","msg":"Compare topology need add observer","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone","OBZone":{"name":"obcluster-1-zone1","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone1","reconcileID":"5d66b681-6458-4bd4-a3d3-9cdffc33838f"}
{"level":"INFO","ts":"2026-04-02T15:28:44+08:00","msg":"Modify obzone replica","controller":"obcluster","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBCluster","OBCluster":{"name":"obcluster","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster","reconcileID":"f1dbc770-e2e7-460b-b1ef-d4b2db8c62af","obzone":"zone3"}
{"level":"INFO","ts":"2026-04-02T15:28:44+08:00","msg":"Compare topology need add observer","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone","OBZone":{"name":"obcluster-1-zone2","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone2","reconcileID":"32eba989-c040-4c65-b4ca-a5fd1897e16d"}
{"level":"INFO","ts":"2026-04-02T15:28:44+08:00","msg":"Compare topology need add observer","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone","OBZone":{"name":"obcluster-1-zone3","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone3","reconcileID":"cb2bdb9a-21dc-4247-b905-df506ab324e6"}
{"level":"INFO","ts":"2026-04-02T15:28:46+08:00","msg":"Create observers","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone","OBZone":{"name":"obcluster-1-zone1","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone1","reconcileID":"dc797fee-2864-4651-b2c4-f272e623a129"}
{"level":"INFO","ts":"2026-04-02T15:28:46+08:00","msg":"Create observer","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone","OBZone":{"name":"obcluster-1-zone1","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone1","reconcileID":"dc797fee-2864-4651-b2c4-f272e623a129","server":"obcluster-1-zone1-5p84x4"}
{"level":"INFO","ts":"2026-04-02T15:28:46+08:00","msg":"Newly created server, init status","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone1-5p84x4","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone1-5p84x4","reconcileID":"57846ddf-92ca-4880-b2a1-ed53a5c36d8f"}
{"level":"INFO","ts":"2026-04-02T15:28:46+08:00","msg":"Create observers","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone","OBZone":{"name":"obcluster-1-zone2","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone2","reconcileID":"2e41643e-4e48-480d-bdbc-d892101ac8bb"}
{"level":"INFO","ts":"2026-04-02T15:28:46+08:00","msg":"Create observer","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone","OBZone":{"name":"obcluster-1-zone2","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone2","reconcileID":"2e41643e-4e48-480d-bdbc-d892101ac8bb","server":"obcluster-1-zone2-vvqjp5"}
{"level":"INFO","ts":"2026-04-02T15:28:46+08:00","msg":"Create observers","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone","OBZone":{"name":"obcluster-1-zone3","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone3","reconcileID":"8e400024-9b86-4505-9a81-5229121191f6"}
{"level":"INFO","ts":"2026-04-02T15:28:46+08:00","msg":"Create observer","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone","OBZone":{"name":"obcluster-1-zone3","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone3","reconcileID":"8e400024-9b86-4505-9a81-5229121191f6","server":"obcluster-1-zone3-k7vp48"}
{"level":"INFO","ts":"2026-04-02T15:28:49+08:00","msg":"Newly created server, init status","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone2-vvqjp5","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone2-vvqjp5","reconcileID":"eac2ef74-1b2d-4b9d-91d9-22723e9eacb6"}
{"level":"INFO","ts":"2026-04-02T15:28:52+08:00","msg":"Newly created server, init status","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone3-k7vp48","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone3-k7vp48","reconcileID":"3bf3f0e7-af32-4426-adea-25189f4600ab"}
{"level":"INFO","ts":"2026-04-02T15:28:55+08:00","msg":"Create observer when obcluster already exists","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone1-5p84x4","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone1-5p84x4","reconcileID":"6281f2fc-e20c-45c8-9196-4797f9264ec4"}
{"level":"INFO","ts":"2026-04-02T15:28:58+08:00","msg":"Create observer when obcluster already exists","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone2-vvqjp5","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone2-vvqjp5","reconcileID":"2179f5fd-c553-4a85-8eb2-3e8d34038b9e"}
{"level":"INFO","ts":"2026-04-02T15:29:01+08:00","msg":"Create observer when obcluster already exists","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone3-k7vp48","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone3-k7vp48","reconcileID":"f2412890-1d62-4509-b5f2-ecc06ab7022c"}
{"level":"INFO","ts":"2026-04-02T15:29:04+08:00","msg":"Create observer service","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone1-5p84x4","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone1-5p84x4","reconcileID":"1e3eb848-de2a-44d5-a12d-065f5f915382"}
{"level":"INFO","ts":"2026-04-02T15:29:07+08:00","msg":"Create observer service","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone2-vvqjp5","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone2-vvqjp5","reconcileID":"a55c1970-dd15-445c-b842-a263d05add06"}
{"level":"INFO","ts":"2026-04-02T15:29:10+08:00","msg":"Create observer service","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone3-k7vp48","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone3-k7vp48","reconcileID":"4e3b9383-a398-453c-8c51-8084748a7897"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Stopping and waiting for non leader election runnables"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"shutting down server","path":"/metrics","kind":"metrics","addr":"127.0.0.1:8080"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Stopping and waiting for leader election runnables"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obtenantvariable","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenantVariable"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obtenantbackuppolicy","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenantBackupPolicy"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obclusteroperation","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBClusterOperation"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obtenantrestore","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenantRestore"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obclusteroperation","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBClusterOperation"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obtenantoperation","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenantOperation"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obtenantrestore","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenantRestore"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obtenantoperation","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenantOperation"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obtenantbackuppolicy","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenantBackupPolicy"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obparameter","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBParameter"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obtenantbackup","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenantBackup"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obtenantbackup","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenantBackup"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obtenantvariable","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenantVariable"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obtenant","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenant"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obtenant","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBTenant"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obcluster","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBCluster"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obcluster","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBCluster"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obparameter","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBParameter"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obzone","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBZone"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"obresourcerescue","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBResourceRescue"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"obresourcerescue","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBResourceRescue"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"Shutdown signal received, waiting for all workers to finish","controller":"k8scluster","controllerGroup":"k8s.oceanbase.com","controllerKind":"K8sCluster"}
{"level":"INFO","ts":"2026-04-02T15:29:33+08:00","msg":"All workers finished","controller":"k8scluster","controllerGroup":"k8s.oceanbase.com","controllerKind":"K8sCluster"}
{"level":"ERROR","ts":"2026-04-02T15:29:34+08:00","msg":"Got error when update observer status","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone1-5p84x4","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone1-5p84x4","reconcileID":"b82436ec-81d7-4780-9010-29723693f2ac","error":"client rate limiter Wait returned an error: context canceled","stacktrace":"github.com/oceanbase/ob-operator/internal/resource/observer.(*OBServerManager).UpdateStatus\n\t/workspace/internal/resource/observer/observer_manager.go:160\ngithub.com/oceanbase/ob-operator/pkg/coordinator.(*Coordinator).Coordinate\n\t/workspace/pkg/coordinator/coordinator.go:96\ngithub.com/oceanbase/ob-operator/internal/controller.(*OBServerReconciler).Reconcile\n\t/workspace/internal/controller/observer_controller.go:108\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
{"level":"ERROR","ts":"2026-04-02T15:29:34+08:00","msg":"Failed to update status","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone1-5p84x4","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone1-5p84x4","reconcileID":"b82436ec-81d7-4780-9010-29723693f2ac","error":"client rate limiter Wait returned an error: context canceled","stacktrace":"github.com/oceanbase/ob-operator/pkg/coordinator.(*Coordinator).Coordinate\n\t/workspace/pkg/coordinator/coordinator.go:98\ngithub.com/oceanbase/ob-operator/internal/controller.(*OBServerReconciler).Reconcile\n\t/workspace/internal/controller/observer_controller.go:108\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
{"level":"ERROR","ts":"2026-04-02T15:29:34+08:00","msg":"Reconciler error","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone1-5p84x4","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone1-5p84x4","reconcileID":"b82436ec-81d7-4780-9010-29723693f2ac","error":"client rate limiter Wait returned an error: context canceled","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}

{"level":"ERROR","ts":"2026-04-02T15:29:37+08:00","msg":"Got error when update observer status","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone2-vvqjp5","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone2-vvqjp5","reconcileID":"310a5155-5e18-49de-ab55-5e8e2771caf0","error":"client rate limiter Wait returned an error: context canceled","stacktrace":"github.com/oceanbase/ob-operator/internal/resource/observer.(*OBServerManager).UpdateStatus\n\t/workspace/internal/resource/observer/observer_manager.go:160\ngithub.com/oceanbase/ob-operator/pkg/coordinator.(*Coordinator).Coordinate\n\t/workspace/pkg/coordinator/coordinator.go:96\ngithub.com/oceanbase/ob-operator/internal/controller.(*OBServerReconciler).Reconcile\n\t/workspace/internal/controller/observer_controller.go:108\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
{"level":"ERROR","ts":"2026-04-02T15:29:37+08:00","msg":"Failed to update status","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone2-vvqjp5","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone2-vvqjp5","reconcileID":"310a5155-5e18-49de-ab55-5e8e2771caf0","error":"client rate limiter Wait returned an error: context canceled","stacktrace":"github.com/oceanbase/ob-operator/pkg/coordinator.(*Coordinator).Coordinate\n\t/workspace/pkg/coordinator/coordinator.go:98\ngithub.com/oceanbase/ob-operator/internal/controller.(*OBServerReconciler).Reconcile\n\t/workspace/internal/controller/observer_controller.go:108\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
{"level":"ERROR","ts":"2026-04-02T15:29:37+08:00","msg":"Reconciler error","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone2-vvqjp5","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone2-vvqjp5","reconcileID":"310a5155-5e18-49de-ab55-5e8e2771caf0","error":"client rate limiter Wait returned an error: context canceled","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
{"level":"ERROR","ts":"2026-04-02T15:29:40+08:00","msg":"Got error when update observer status","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone3-k7vp48","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone3-k7vp48","reconcileID":"8a75d461-c278-4e8a-be16-69e6231e663f","error":"client rate limiter Wait returned an error: context canceled","stacktrace":"github.com/oceanbase/ob-operator/internal/resource/observer.(*OBServerManager).UpdateStatus\n\t/workspace/internal/resource/observer/observer_manager.go:160\ngithub.com/oceanbase/ob-operator/pkg/coordinator.(*Coordinator).Coordinate\n\t/workspace/pkg/coordinator/coordinator.go:96\ngithub.com/oceanbase/ob-operator/internal/controller.(*OBServerReconciler).Reconcile\n\t/workspace/internal/controller/observer_controller.go:108\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
{"level":"ERROR","ts":"2026-04-02T15:29:40+08:00","msg":"Failed to update status","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone3-k7vp48","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone3-k7vp48","reconcileID":"8a75d461-c278-4e8a-be16-69e6231e663f","error":"client rate limiter Wait returned an error: context canceled","stacktrace":"github.com/oceanbase/ob-operator/pkg/coordinator.(*Coordinator).Coordinate\n\t/workspace/pkg/coordinator/coordinator.go:98\ngithub.com/oceanbase/ob-operator/internal/controller.(*OBServerReconciler).Reconcile\n\t/workspace/internal/controller/observer_controller.go:108\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:314\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
{"level":"ERROR","ts":"2026-04-02T15:29:40+08:00","msg":"Reconciler error","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer","OBServer":{"name":"obcluster-1-zone3-k7vp48","namespace":"test-oceanbase"},"namespace":"test-oceanbase","name":"obcluster-1-zone3-k7vp48","reconcileID":"8a75d461-c278-4e8a-be16-69e6231e663f","error":"client rate limiter Wait returned an error: context canceled","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:265\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226"}
{"level":"INFO","ts":"2026-04-02T15:29:40+08:00","msg":"All workers finished","controller":"observer","controllerGroup":"oceanbase.oceanbase.com","controllerKind":"OBServer"}
{"level":"INFO","ts":"2026-04-02T15:29:40+08:00","msg":"Stopping and waiting for caches"}
{"level":"INFO","ts":"2026-04-02T15:29:40+08:00","msg":"Stopping and waiting for webhooks"}
{"level":"INFO","ts":"2026-04-02T15:29:40+08:00","logger":"controller-runtime.webhook","msg":"Shutting down webhook server with timeout of 1 minute"}
{"level":"INFO","ts":"2026-04-02T15:29:40+08:00","msg":"Wait completed, proceeding to shutdown the manager"}
{"level":"ERROR","ts":"2026-04-02T15:29:40+08:00","msg":"error received after stop sequence was engaged","error":"leader election lost","stacktrace":"sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).engageStopProcedure.func1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/internal.go:555"}
rpc error: code = Unknown desc = Error response from daemon: No such container: c69cdfd831487ed43852aa95929bf9184401cddd100ee5b7eedb87ba2ef8e2dd