【 使用环境 】测试环境
【 OB or 其他组件 】ob
【 使用版本 】4.3.0
【问题描述】NFS 作为归档介质时使用系统租户为指定租户设置路径报错ERROR 4009 (58030): IO error
【附件及日志】
select last_Trace_id();
±----------------------------------+
| last_Trace_id() |
±----------------------------------+
| YB4A0A000407-00061BF1D17832A0-0-0 |
±----------------------------------+
1 row in set (0.001 sec)
grep YB4A0A000407-00061BF1D17832A0-0-0 *
$ grep YB4A0A000407-00061BF1D17832A0-0-0 *
observer.log:[2024-07-30 12:31:17.867658] WDIAG [RPC] send (ob_poc_rpc_proxy.h:173) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=14][errcode=-4009] execute rpc fail(addr=“10.0.4.7:2890”, pcode=648, ret=-4009, timeout=9999839)
observer.log:[2024-07-30 12:31:17.867672] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:244) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=12][errcode=-4009]
observer.log:[2024-07-30 12:31:17.867679] WDIAG [SQL.ENG] execute (ob_alter_system_executor.cpp:1241) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3][errcode=-4009] set config rpc failed(ret=-4009, rpc_arg={items:[{name:“log_archive_dest”, value:“location=file:///mnt/oceanbasedata/bddb/archive”, comment:"", zone:"", server:“0.0.0.0:0”, tenant_name:“bddb”, exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}], is_inner:false})
observer.log:[2024-07-30 12:31:17.867706] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=9] event table add task(ret=0, event_table_name=“all_server_event_history", sql=INSERT INTO all_server_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, name4, value4, value5, value6, svr_ip, svr_port) VALUES (usec_to_time(1722313877867689), ‘sql’, ‘execute_cmd’, ‘cmd_type’, 177, ‘sql_text’, X’2A2A2A’, ‘return_code’, -4009, ‘tenant_id’, 1, ‘’, ‘’, ‘10.0.4.7’, 2890))
observer.log:[2024-07-30 12:31:17.867716] WDIAG [SQL] open_cmd (ob_result_set.cpp:102) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=9][errcode=-4009] execute cmd failed(ret=-4009)
observer.log:[2024-07-30 12:31:17.867720] WDIAG [SQL] open (ob_result_set.cpp:161) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3][errcode=-4009] execute plan failed(ret=-4009)
observer.log:[2024-07-30 12:31:17.867724] WDIAG [SERVER] response_result (ob_sync_cmd_driver.cpp:143) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3][errcode=-4009] close result set fail(cret=-4009)
observer.log:[2024-07-30 12:31:17.867729] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:986) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3][errcode=-4009] [RETRY] check if need retry(v={force_local_retry:false, stmt_retry_times:0, local_retry_times:0, err:-4009, err:“OB_IO_ERROR”, retry_type:0, client_ret:-4009}, need_retry=false)
observer.log:[2024-07-30 12:31:17.867735] WDIAG [SERVER] response_result (ob_sync_cmd_driver.cpp:149) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=6][errcode=-4009] result set open failed, check if need retry(ret=-4009, cli_ret=-4009, retry_ctrl.need_retry()=0)
observer.log:[2024-07-30 12:31:17.867748] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:378) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=4] sending error packet(ob_error=-4009, client error=-4009, extra_err_info=NULL, lbt()=“0x17a30451 0xb6da170 0xb681d45 0x5ea8f77 0x5c37bcb 0x5c2b6ce 0x18b90701 0x5c24fb0 0x5c1f72a 0xb3c3824 0x17b1814e 0x2b20b6789ea5 0x2b20b6a9bf3d”)
observer.log:[2024-07-30 12:31:17.867762] WDIAG [SERVER] do_process (obmp_query.cpp:818) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=5][errcode=-4009] execute query fail(ret=-4009, timeout_timestamp=1722313887865962)
observer.log:[2024-07-30 12:31:17.867783] WDIAG [SERVER.OMT] process_one (ob_worker_processor.cpp:89) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3][errcode=-4009] process request fail(ret=-4009)
observer.log:[2024-07-30 12:31:17.867787] WDIAG [SERVER.OMT] process (ob_worker_processor.cpp:157) [69523][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3][errcode=-4009] process request fail(ret=-4009)
rootservice.log:[2024-07-30 12:31:17.867102] WDIAG [SHARE] parse_one_row (ob_inner_table_operator.cpp:947) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=11][errcode=-4018] no row exist(ret=-4018)
rootservice.log:[2024-07-30 12:31:17.867112] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:620) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=9][errcode=-4018] failed to parse row(ret=-4018, sql=select * from all_log_archive_progress where tenant_id = 1004 AND dest_no = 0, exec_tenant_id=1003)
rootservice.log:[2024-07-30 12:31:17.867126] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1004, dest_no:0})
rootservice.log:[2024-07-30 12:31:17.867132] WDIAG [SHARE] get_round (ob_archive_persist_helper.cpp:488) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=6][errcode=-4018] failed to get round(ret=-4018)
rootservice.log:[2024-07-30 12:31:17.867178] INFO list_directories (ob_storage_file.cpp:681) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=4] list directories count(dir_path=/mnt/oceanbasedata/bddb/archive/, ret=0)
rootservice.log:[2024-07-30 12:31:17.867207] INFO [STORAGE] mkdir (ob_storage_file.cpp:268) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=8] path exist(pos=31, path=/mnt/oceanbasedata/bddb/archive)
rootservice.log:[2024-07-30 12:31:17.867224] WDIAG [STORAGE] mkdir (ob_storage_file.cpp:304) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=7][errcode=-4009] failed to create parent dir(ret=-4009, path=/mnt/oceanbasedata/bddb/archive/check_file, errno=13, errno=“Permission denied”)
rootservice.log:[2024-07-30 12:31:17.867232] WDIAG [STORAGE] mkdir (ob_storage.cpp:1193) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=8][errcode=-4009] failed to mkdir(ret=-4009, uri=file:///mnt/oceanbasedata/bddb/archive/check_file)
rootservice.log:[2024-07-30 12:31:17.867236] WDIAG mkdir (ob_backup_io_adapter.cpp:219) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=4][errcode=-4009] fail to mkdir!(ret=-4009, uri=file:///mnt/oceanbasedata/bddb/archive/check_file, storage_info=0x2b219d31a470)
rootservice.log:[2024-07-30 12:31:17.867244] WDIAG [SHARE] create_check_file_dir (ob_backup_connectivity.cpp:304) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=7][errcode=-4009] failed to check is exist(ret=-4009, path={cur_pos:49, path:“file:///mnt/oceanbasedata/bddb/archive/check_file”}, backup_dest={root_path:“file:///mnt/oceanbasedata/bddb/archive”, storage_info:{endpoint:"", access_id:"", extension:"", type:“FILE”, checksum_type:1}})
rootservice.log:[2024-07-30 12:31:17.867261] WDIAG [SHARE] create_connectivity_check_file (ob_backup_connectivity.cpp:409) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=16][errcode=-4009] failed to get check file path(ret=-4009, backup_dest={root_path:“file:///mnt/oceanbasedata/bddb/archive”, storage_info:{endpoint:"", access_id:"", extension:"", type:“FILE”, checksum_type:1}})
rootservice.log:[2024-07-30 12:31:17.867269] WDIAG [SHARE] prepare_connectivity_check_file (ob_backup_connectivity.cpp:157) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=7][errcode=-4009] failed to create check file(ret=-4009, tenant_id=1004)
rootservice.log:[2024-07-30 12:31:17.867272] WDIAG [SHARE] check_backup_dest_connectivity (ob_backup_connectivity.cpp:202) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3][errcode=-4009] failed to prepare check file(ret=-4009, tenant_id=1004, backup_dest={root_path:“file:///mnt/oceanbasedata/bddb/archive”, storage_info:{endpoint:”", access_id:"", extension:"", type:“FILE”, checksum_type:1}})
rootservice.log:[2024-07-30 12:31:17.867280] WDIAG [SHARE] check_dest_connectivity (ob_backup_store.cpp:417) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=7][errcode=-4009] fail to check backup dest connectivity(ret=-4009, tenant_id=1004)
rootservice.log:[2024-07-30 12:31:17.867283] WDIAG [SHARE] check_dest_validity (ob_backup_store.cpp:437) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3][errcode=-4009] fail to check dest connectivity(ret=-4009, backup_dest={root_path:“file:///mnt/oceanbasedata/bddb/archive”, storage_info:{endpoint:"", access_id:"", extension:"", type:“FILE”, checksum_type:1}})
rootservice.log:[2024-07-30 12:31:17.867287] WDIAG [SHARE] check_before_update_inner_config (ob_backup_config.cpp:663) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=4][errcode=-4009] fail to update archive dest config(ret=-4009, tenant_id=1004)
rootservice.log:[2024-07-30 12:31:17.867294] WDIAG [SHARE] update_inner_config_table (ob_backup_config.cpp:367) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=6][errcode=-4009] fail to check before update inner config(ret=-4009)
rootservice.log:[2024-07-30 12:31:17.867297] WDIAG [RS] admin_set_backup_config (ob_root_service.cpp:11217) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3][errcode=-4009] fail to update inner config table(ret=-4009)
rootservice.log:[2024-07-30 12:31:17.867332] INFO [STORAGE.TRANS] rollback_tx (ob_tx_api.cpp:365) [69537][T1_L0_G0][T1003][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3] rollback tx(ret=0, *this={is_inited_:true, tenant_id_:1003, this:0x2b2123204030}, tx={this:0x2b21ba6061f0, tx_id:{txid:80198356}, state:7, addr:“10.0.4.7:2890”, tenant_id:1003, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:“0.0.0.0:0”, access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:{branch:0, seq:1}, op_sn:3, alloc_ts:1722313877865955, active_ts:1722313877865955, commit_ts:-1, finish_ts:1722313877867017, timeout_us:9999085, lock_timeout_us:-1, expire_ts:1722313887865040, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:1717559948, cluster_version:17180066048, seq_base:1722313877878139, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:-6002, commit_expire_ts:0, commit_task_.is_registered():false, ref:2})
rootservice.log:[2024-07-30 12:31:17.867400] WDIAG [RS] admin_set_config (ob_root_service.cpp:8840) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=29][errcode=-4009] fail to set backup config(ret=-4009, arg={items:[{name:“log_archive_dest”, value:“location=file:///mnt/oceanbasedata/bddb/archive”, comment:"", zone:"", server:“0.0.0.0:0”, tenant_name:“bddb”, exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}], is_inner:false})
rootservice.log:[2024-07-30 12:31:17.867437] INFO [SHARE] add_event (ob_event_history_table_operator.h:261) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=12] event table add task(ret=0, event_table_name="__all_rootservice_event_history", sql=INSERT INTO all_rootservice_event_history (gmt_create, module, event, name1, value1, name2, value2, name3, value3, value4, value5, value6, rs_svr_ip, rs_svr_port) VALUES (usec_to_time(1722313877867413), ‘root_service’, ‘admin_set_config’, ‘ret’, -4009, ‘arg’, ‘{name:“log_archive_dest”, value:“location=file:///mnt/oceanbasedata/bddb/archive”, comment:"", zone:"", server:“0.0.0.0:0”, tenant_name:“bddb”, exec_tenant_id:1, tenant_ids:[], want_to_set_tenant_config:false}’, ‘is_inner’, 0, ‘’, ‘’, ‘’, ‘10.0.4.7’, 2890))
rootservice.log:[2024-07-30 12:31:17.867444] WDIAG [RS] process (ob_rs_rpc_processor.h:212) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=6][errcode=-4009] process failed(ret=-4009)
rootservice.log:[2024-07-30 12:31:17.867448] INFO [RS] process (ob_rs_rpc_processor.h:232) [69537][T1_L0_G0][T1][YB4A0A000407-00061BF1D17832A0-0-0] [lt=3] [DDL] execute ddl like stmt(ret=-4009, cost=1045, ddl_arg=NULL)
请教一下这个报错是什么意思呢?
