top
top - 13:14:31 up 3:14, 4 users, load average: 27.12, 29.08, 23.66
Tasks: 269 total, 1 running, 268 sleeping, 0 stopped, 0 zombie
%Cpu(s): 29.1 us, 9.5 sy, 0.0 ni, 16.7 id, 0.0 wa, 41.6 hi, 3.1 si, 0.0 st
MiB Mem : 10665.3 total, 7046.3 free, 1843.3 used, 1775.7 buff/cache
MiB Swap: 4096.0 total, 4096.0 free, 0.0 used. 4816.1 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
4841 admin 20 0 2332368 1.8g 400088 S 294.1 17.3 20:22.85 observer
obclient(root@sys)[oceanbase]> select CON_ID,WAIT_CLASS,EVENT,TIME_WAITED from GV$SYSTEM_EVENT order by TIME_WAITED desc;
+--------+---------------+-----------------------------------------------------------+-------------+
| CON_ID | WAIT_CLASS | EVENT | TIME_WAITED |
+--------+---------------+-----------------------------------------------------------+-------------+
| 1 | IDLE | sleep wait | 383217.8055 |
| 1 | OTHER | exec inner sql wait | 143233.9061 |
| 1 | NETWORK | sync rpc | 64488.7869 |
| 1 | CONCURRENCY | default condition wait | 49561.4665 |
| 1 | SYSTEM_IO | palf write | 37622.3568 |
| 1 | CONCURRENCY | sleep: periodic task throttle wait | 32108.1623 |
| 1 | CONCURRENCY | latch: refresh schema lock wait | 19393.6589 |
| 1 | SYSTEM_IO | db file compact read | 17509.2112 |
| 1 | NETWORK | async rpc proxy condition wait | 7830.3863 |
| 1 | CONCURRENCY | latch: rs broadcast lock wait | 7108.385 |
| 1 | CONFIGURATION | wait in request queue | 5053.1766 |
| 1 | CONCURRENCY | latch: default bucket lock wait | 4530.7759 |
| 1 | USER_IO | db file data read | 3658.8306 |
| 1 | NETWORK | sync get gts timestamp wait | 2807.503 |
| 1 | COMMIT | tx commiting wait | 1976.6958 |
| 1 | CONCURRENCY | latch: tenant meta memory manager lock wait | 632.8688 |
| 1 | CONCURRENCY | latch: default spin rwlock wait | 478.6119 |
| 1 | SYSTEM_IO | palf read | 271.9454 |
| 1 | CONCURRENCY | sleep: lock for read need wait for concurrency control | 209.3922 |
| 1 | CONCURRENCY | latch: config lock wait | 157.6327 |
| 1 | CONCURRENCY | latch: palf handle impl lock wait | 123.9762 |
[2026-02-27 12:59:34.810216] WDIAG [OCCAM] operator() (ob_occam_timer.h:81) [2405][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=344][errcode=-4016] task executed too delay(delay_time=65465, *p_task_={this:0x7efca2c06630, caller:ob_failure_detector.cpp:mtl_start:78, function_type:ZN9oceanbase10logservice11coordinator17ObFailureDetector9mtl_startERPS2_E3$_6, timer_running_flag:1, total_running_count:2, func_shared_ptr_.ptr:0x7efc7dc94e10, time_wheel:0x7efc7b1dd990, thread_pool:0x7efc7b1dd170, is_running:0, time_interval:1.00s, expected_run_time:12:59:35.636, task_priority:1, with_handle_protected:1})
[2026-02-27 12:59:35.506052] WDIAG [STORAGE] load_arb_service_info (ob_locality_manager.cpp:264) [2290][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=792][errcode=-4747] fail to get arbitration service info(ret=-4747, ret="OB_ARBITRATION_SERVICE_NOT_EXIST", arbitration_service_key=default, lock_line=false, arb_service_info={arbitration_service_key:, arbitration_service:, previous_arbitration_service:, type:{type:-1}})
[2026-02-27 12:59:35.507096] WDIAG [STORAGE] process (ob_locality_manager.cpp:763) [2290][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=427][errcode=-4747] load_arb_service_info fail(ret=-4747)
[2026-02-27 12:59:35.689310] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [2286][VTblLocAsyncUp0][T0][YB42C0A802E3-00064BC4C85D8346-0-0] [lt=1046] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12287, locations=[{table_id:12287, partition_id:0, partition_cnt:1, replica_locations:[{server:"192.168.2.227:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1772168375689148, sql_renew_time:0, is_mark_fail:false}])
[2026-02-27 12:59:35.689772] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [2286][VTblLocAsyncUp0][T0][YB42C0A802E3-00064BC4C85D8346-0-0] [lt=437] success to process renew task(task={tenant_id:1, table_id:12287, add_timestamp:1772168375676682}, locations=[{table_id:12287, partition_id:0, partition_cnt:1, replica_locations:[{server:"192.168.2.227:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1772168375689148, sql_renew_time:0, is_mark_fail:false}])
[2026-02-27 12:59:35.693127] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [2594][BlackListServic][T1][YB42C0A802E3-00064BC4C4FD80E9-0-0] [lt=430][errcode=-4008] execute fail(ret=-4008)
[2026-02-27 12:59:36.842227] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:262) [2546][T1_ArcSender][T1][YB42C0A802E3-00064BC4C49D73F0-0-0] [lt=772][errcode=-4018] try free send task failed(ret=-4018)
[2026-02-27 12:59:37.647014] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1041) [2539][T1_Flush][T1][Y0-0000000000000000-0-0] [lt=1456] get_min_unreplayed_log_info(lsn={lsn:98611414}, scn={val:1772103752737895001, v:0}, this={ls_id_:{id:1}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1772158425166410, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10850054, ret_code_:0}}, next_to_submit_lsn_:{lsn:98611414}, next_to_submit_scn_:{val:1772103752737895001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1772102202432411001, v:0}, iterator_:{iterator_impl:{buf_:0x7efc60a05000, next_round_pread_size:3678208, curr_read_pos:2118524, curr_read_buf_start_pos:0, curr_read_buf_end_pos:2118524, log_storage_:{IteratorStorage:{start_lsn:{lsn:96492890}, end_lsn:{lsn:98611414}, read_buf:{buf_len_:3682304, buf_:0x7efc60a05000}, block_size:67104768, log_storage_:0x7efc6c02a4f0, io_ctx:{user:"REPLAY"}}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1772103752737895000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:2, log_size:500, scn_:{val:1772103752737895000, v:0}, data_checksum:1579399240, flag:794}}, init_mode_version:0, accumulate_checksum:4158142229, curr_entry_is_padding:0, padding_entry_size:532, padding_entry_scn:{val:1772103752737895000, v:0}}}}})
[2026-02-27 12:59:38.433117] WDIAG [SHARE] fixup_last_stat (ob_active_session_guard.cpp:42) [2131][SyslogCompress][T0][Y0-0000000000000000-0-0] [lt=755][errcode=-4016] fixup index with no fixup buffer.(fixup_index=2256, this={ObActiveSessionStatItem:{tenant_id:1, user_id:0, session_id:2305843009213693961, event id:15102, event:"sleep wait", wait_time:0, time_model:0, trace_id:Y0-0000000000000000-0-0, plan_line_id:-1, sql_id:"", top_level_sql_id:"", plsql_entry_subprogram_name:"", plsql_subprogram_name:"", session_type:1, is_wr_sample:false, delta_time:1010850, delta_cpu_time:0, delta_db_time:0, program:"SyslogCompress", module:"SyslogCompress", action:"", client_id:"", group_id:0, tid:2131, plan_hash:0, tx_id:0, stmt_type:0, tablet_id:0, block_sessid:0, delta_read:{count:0, size:0}, delta_write:{count:0, size:0}}, last_touch_ts:29040523385521, wait_event_begin_ts:29040523385521, total_idle_wait_time:10014732454, total_non_idle_wait_time:0, prev_idle_wait_time:10013878643, prev_non_idle_wait_time:0, total_cpu_time:0, is_active_session:true, inner_sql_wait_type_id:0, pcode:0, tm_idle_cpu_cycles:0, fixup_index:2256, fixup_ash_buffer:{this:0x7efca2bafbf0, block_ptr_.control_ptr:null, block_ptr_.data_ptr:null}, retry_wait_event_no:0, retry_plan_line_id:-1, flags:0, mysql_cmd:-1})
[2026-02-27 12:59:38.693610] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [2594][BlackListServic][T1][YB42C0A802E3-00064BC4C4FD80EA-0-0] [lt=849][errcode=-4008] execute fail(ret=-4008)
[2026-02-27 12:59:39.100981] INFO [CLOG] get_replay_process (ob_replay_status.cpp:1078) [2506][T1_ReplayProces][T1][Y0-0000000000000000-0-0] [lt=906] replay status is not follower(min_unreplayed_lsn={lsn:98611414}, base_lsn={lsn:0}, this={ls_id_:{id:1}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:1, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1772158425166410, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:10850054, ret_code_:0}}, next_to_submit_lsn_:{lsn:98611414}, next_to_submit_scn_:{val:1772103752737895001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1772102202432411001, v:0}, iterator_:{iterator_impl:{buf_:0x7efc60a05000, next_round_pread_size:3678208, curr_read_pos:2118524, curr_read_buf_start_pos:0, curr_read_buf_end_pos:2118524, log_storage_:{IteratorStorage:{start_lsn:{lsn:96492890}, end_lsn:{lsn:98611414}, read_buf:{buf_len_:3682304, buf_:0x7efc60a05000}, block_size:67104768, log_storage_:0x7efc6c02a4f0, io_ctx:{user:"REPLAY"}}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1772103752737895000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:2, log_size:500, scn_:{val:1772103752737895000, v:0}, data_checksum:1579399240, flag:794}}, init_mode_version:0, accumulate_checksum:4158142229, curr_entry_is_padding:0, padding_entry_size:532, padding_entry_scn:{val:1772103752737895000, v:0}}}}})
[2026-02-27 12:59:39.831056] WDIAG [OCCAM] operator() (ob_occam_timer.h:81) [2405][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=395][errcode=-4016] task executed too delay(delay_time=77401, *p_task_={this:0x7efca2c06630, caller:ob_failure_detector.cpp:mtl_start:78, function_type:ZN9oceanbase10logservice11coordinator17ObFailureDetector9mtl_startERPS2_E3$_6, timer_running_flag:1, total_running_count:2, func_shared_ptr_.ptr:0x7efc7dc94e10, time_wheel:0x7efc7b1dd990, thread_pool:0x7efc7b1dd170, is_running:0, time_interval:1.00s, expected_run_time:12:59:40.636, task_priority:1, with_handle_protected:1})
[2026-02-27 12:59:41.684185] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [2286][VTblLocAsyncUp0][T0][YB42C0A802E3-00064BC4C85D8347-0-0] [lt=993] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12239, locations=[{table_id:12239, partition_id:12239, partition_cnt:0, replica_locations:[{server:"192.168.2.227:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1772168381683429, sql_renew_time:1772168381683429, is_mark_fail:false}])
[2026-02-27 12:59:41.684675] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [2286][VTblLocAsyncUp0][T0][YB42C0A802E3-00064BC4C85D8347-0-0] [lt=469] success to process renew task(task={tenant_id:1, table_id:12239, add_timestamp:1772168381682692}, locations=[{table_id:12239, partition_id:12239, partition_cnt:0, replica_locations:[{server:"192.168.2.227:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1772168381683429, sql_renew_time:1772168381683429, is_mark_fail:false}])
[2026-02-27 12:59:41.697331] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_ls_info.cpp:54) [2594][BlackListServic][T1][YB42C0A802E3-00064BC4C4FD80EB-0-0] [lt=383][errcode=-4008] execute fail(ret=-4008)
[2026-02-27 12:59:41.729146] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [2489][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=772][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1 AND dest_no = 0 AND name = 'dest_id' for update, exec_tenant_id=1)
[2026-02-27 12:59:41.730270] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [2489][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=197][errcode=-4018] failed to get row(ret=-4018, need_lock=true, key={tenant_id:1, *item:{dest_no:0, name:"dest_id", *value:{value:0}}})
[2026-02-27 12:59:41.730499] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [2489][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=204][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1, *item:{dest_no:0, name:"dest_id", *value:{value:0}}}, need_lock=true)
[2026-02-27 12:59:41.731200] WDIAG [SHARE] get_dest_id (ob_archive_persist_helper.cpp:223) [2489][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=204][errcode=-4018] failed to get item(ret=-4018, need_lock=true, item_with_tenant_id={tenant_id:1, *item:{dest_no:0, name:"dest_id", *value:{value:0}}})
[2026-02-27 12:59:41.739224] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:623) [2489][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=815][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_backup_parameter where tenant_id = 1 AND name = 'data_backup_dest', exec_tenant_id=1)
[2026-02-27 12:59:41.740144] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:172) [2489][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=199][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1, *item:{name:"data_backup_dest", *value:{value:}}})
[2026-02-27 12:59:41.740376] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:349) [2489][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=209][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1, *item:{name:"data_backup_dest", *value:{value:}}}, need_lock=false)
[2026-02-27 12:59:41.740603] WDIAG [SHARE] get_backup_dest (ob_backup_helper.cpp:71) [2489][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=201][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1, *item:{name:"data_backup_dest", *value:{value:}}})
[2026-02-27 12:59:41.773042] WDIAG read_and_convert_to_tuples_ (ob_table_access_helper.h:449) [2522][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=179][errcode=-4008] iter failed(iter_times=0, ret=-4008, ret="OB_ITER_END", MTL_ID()=1, table=oceanbase.__all_detect_lock_info_v2, condition=WHERE detect_func_no = 1 GROUP BY owner_id)
