load data 导入数据失败

OceanBase
#1

【产品名称】

oceanbase

【产品版本】

3.1.2

【问题描述】

通过TPCH创建100G数据，连接数据库执行 load data /*+ parallel(80) */ infile ‘/lineitem.tbl’ into table lineitem fields terminated by ‘|’; ，进程CRASH ERROR。数据库压缩算法为zstd，日志信息：

[2022-03-29 10:25:44.830565] ERROR [STORAGE] check_micro_block (ob_macro_block_writer.cpp:1679) [45524][448][Y16FA7F000001-0005DB521C5F823A] [lt=77] [dc=0] decompressed size is not equal to original size(ret=-4103, uncompressed_size=16392, real_decomp_size=32898) BACKTRACE:0x99cec2e 0x9744f31 0x22a7e46 0x22a7a8b 0x22a7803 0x229b989 0x8355c64 0x834a354 0x834868f 0x8348276 0x7b64e39 0x7b72a0c 0x7c98726 0x338583a 0x3388cb0 0x236ef0f 0x23a37a2 0x998c535 0x998b262 0x973526f

[2022-03-29 10:25:44.870935] ERROR [STORAGE] check_micro_block (ob_macro_block_writer.cpp:1679) [45524][448][Y16FA7F000001-0005DB521C5F8244] [lt=46] [dc=0] decompressed size is not equal to original size(ret=-4103, uncompressed_size=16457, real_decomp_size=32914) BACKTRACE:0x99cec2e 0x9744f31 0x22a7e46 0x22a7a8b 0x22a7803 0x229b989 0x8355c64 0x834a354 0x834868f 0x8348276 0x7b64e39 0x7b72a0c 0x7c98726 0x338583a 0x3388cb0 0x236ef0f 0x23a37a2 0x998c535 0x998b262 0x973526f

[2022-03-29 10:25:44.870936] ERROR [STORAGE] check_micro_block (ob_macro_block_writer.cpp:1679) [45517][434][Y16FA7F000001-0005DB521C5F8235] [lt=75] [dc=0] decompressed size is not equal to original size(ret=-4103, uncompressed_size=16413, real_decomp_size=16457) BACKTRACE:0x99cec2e 0x9744f31 0x22a7e46 0x22a7a8b 0x22a7803 0x229b989 0x8355c64 0x834a354 0x834868f 0x8348276 0x7b64e39 0x7b72a0c 0x7c98726 0x338583a 0x3388cb0 0x236ef0f 0x23a37a2 0x998c535 0x998b262 0x973526f

[2022-03-29 10:25:44.952473] ERROR [STORAGE] check_micro_block (ob_macro_block_writer.cpp:1679) [45524][448][Y16FA7F000001-0005DB521C5F8249] [lt=51] [dc=0] decompressed size is not equal to original size(ret=-4103, uncompressed_size=16469, real_decomp_size=32768) BACKTRACE:0x99cec2e 0x9744f31 0x22a7e46 0x22a7a8b 0x22a7803 0x229b989 0x8355c64 0x834a354 0x834868f 0x8348276 0x7b64e39 0x7b72a0c 0x7c98726 0x338583a 0x3388cb0 0x236ef0f 0x23a37a2 0x998c535 0x998b262 0x973526f

[2022-03-29 10:25:45.003211] ERROR [STORAGE] check_micro_block (ob_macro_block_writer.cpp:1679) [45524][448][Y16FA7F000001-0005DB521C5F825C] [lt=36] [dc=0] decompressed size is not equal to original size(ret=-4103, uncompressed_size=16386, real_decomp_size=32866) BACKTRACE:0x99cec2e 0x9744f31 0x22a7e46 0x22a7a8b 0x22a7803 0x229b989 0x8355c64 0x834a354 0x834868f 0x8348276 0x7b64e39 0x7b72a0c 0x7c98726 0x338583a 0x3388cb0 0x236ef0f 0x23a37a2 0x998c535 0x998b262 0x973526f

[2022-03-29 10:25:45.043251] ERROR [STORAGE] check_micro_block (ob_macro_block_writer.cpp:1679) [45517][434][Y16FA7F000001-0005DB521C5F8252] [lt=51] [dc=0] decompressed size is not equal to original size(ret=-4103, uncompressed_size=16405, real_decomp_size=16386) BACKTRACE:0x99cec2e 0x9744f31 0x22a7e46 0x22a7a8b 0x22a7803 0x229b989 0x8355c64 0x834a354 0x834868f 0x8348276 0x7b64e39 0x7b72a0c 0x7c98726 0x338583a 0x3388cb0 0x236ef0f 0x23a37a2 0x998c535 0x998b262 0x973526f

[2022-03-29 10:25:45.043250] ERROR [STORAGE] check_micro_block (ob_macro_block_writer.cpp:1679) [45524][448][Y16FA7F000001-0005DB521C5F8261] [lt=29] [dc=0] decompressed size is not equal to original size(ret=-4103, uncompressed_size=16386, real_decomp_size=32772) BACKTRACE:0x99cec2e 0x9744f31 0x22a7e46 0x22a7a8b 0x22a7803 0x229b989 0x8355c64 0x834a354 0x834868f 0x8348276 0x7b64e39 0x7b72a0c 0x7c98726 0x338583a 0x3388cb0 0x236ef0f 0x23a37a2 0x998c535 0x998b262 0x973526f

[2022-03-29 10:25:45.189276] ERROR [STORAGE] check_micro_block (ob_macro_block_writer.cpp:1679) [45524][448][Y16FA7F000001-0005DB521C5F8274] [lt=43] [dc=0] decompressed size is not equal to original size(ret=-4103, uncompressed_size=16388, real_decomp_size=32776) BACKTRACE:0x99cec2e 0x9744f31 0x22a7e46 0x22a7a8b 0x22a7803 0x229b989 0x8355c64 0x834a354 0x834868f 0x8348276 0x7b64e39 0x7b72a0c 0x7c98726 0x338583a 0x3388cb0 0x236ef0f 0x23a37a2 0x998c535 0x998b262 0x973526f

[2022-3-29 2:25:45.189437] INFO [run1@ob_signal_worker.cpp:203] [48846][Y7369676E616C-1] receive request, req=1, accumulated=1

CRASH ERROR!!! sig=11, sig_code=1, sig_addr=7f6da82a9e87, timestamp=1648520745574191, tid=45517, tname=DAG_MINOR_MERGE, trace_id=25265128341505-1648520593375852, extra_info=((null)), lbt=0x99eca68 0x99dc8a8 0x7f759eca1c1f 0x84acc7f 0x83fd673 0x8416897 0x84162bb 0x8415714 0x840db21 0x85a3451 0x85b0d2c 0x858b8f9 0x7b6b4a7 0x7b6f7f3 0x7b72a60 0x7c98725 0x3385839 0x3388caf 0x236ef0e 0x23a37a1 0x998c534 0x998b261 0x973526e

#2

 addr2line -Cfe ./observer 0x99cec2e 0x9744f31 0x22a7e46 0x22a7a8b 0x22a7803 0x229b989 0x8355c64 0x834a354 0x834868f 0x8348276 0x7b64e39 0x7b72a0c 0x7c98726 0x338583a 0x3388cb0 0x236ef0f 0x23a37a2 0x998c535 0x998b262 0x973526f

oceanbase::common::parray(char*, long, long*, int)

./build_release/deps/oblib/src/lib/./deps/oblib/src/lib/utility/utility.cpp:44

oceanbase::common::logdata_print_info(char*, long, long&, char const*)

./build_release/deps/oblib/src/lib/./deps/oblib/src/lib/oblog/ob_log_print_kv.h:302

operator()

./build_release/src/share/./deps/oblib/src/lib/oblog/ob_log.h:807

void oceanbase::common::ObLogger::do_async_log_message<oceanbase::common::ObLogger::log_message_kv(char const*, int, char const*, int, char const*, unsigned long, char const*, char const*, char const*, char const*)::{lambda(oceanbase::common::ObPLogItem*)#1}>(char const*, int, char const*, int, char const*, unsigned long, oceanbase::common::ObLogger::log_message_kv(char const*, int, char const*, int, char const*, unsigned long, char const*, char const*, char const*, char const*)::{lambda(oceanbase::common::ObPLogItem*)#1}&)

./build_release/src/share/./deps/oblib/src/lib/oblog/ob_log.h:1401

oceanbase::common::ObLogger::log_message_kv(char const*, int, char const*, int, char const*, unsigned long, char const*, char const*, char const*, char const*)

??:?

void oceanbase::common::OB_PRINT<int, long, long>(char const*, int, char const*, int, char const*, unsigned long, char const*, char const*, int const&, char const*, long const&, char const*, long const&)

./build_release/src/share/./deps/oblib/src/lib/oblog/ob_log.h:1463

oceanbase::blocksstable::ObMacroBlockWriter::check_micro_block(char const*, long, char const*, long, oceanbase::blocksstable::ObIMicroBlockWriter*)

./build_release/src/storage/./src/storage/blocksstable/ob_macro_block_writer.cpp:1678

oceanbase::blocksstable::ObMacroBlockWriter::build_micro_block(bool)

./build_release/src/storage/./src/storage/blocksstable/ob_macro_block_writer.cpp:903

oceanbase::blocksstable::ObMacroBlockWriter::append_row(oceanbase::storage::ObStoreRow const&, long, bool)

./build_release/src/storage/./src/storage/blocksstable/ob_macro_block_writer.cpp:317

oceanbase::blocksstable::ObMacroBlockWriter::append_row(oceanbase::storage::ObStoreRow const&, bool)

./build_release/src/storage/./src/storage/blocksstable/ob_macro_block_writer.cpp:254

oceanbase::compaction::ObMacroBlockBuilder::process(oceanbase::storage::ObStoreRow const&, oceanbase::compaction::ObCompactRowType::ObCompactRowTypeEnum)

./build_release/src/storage/./src/storage/compaction/ob_partition_merge_builder.cpp:612

oceanbase::compaction::ObPartitionMergeUtil::merge_partition(oceanbase::memtable::ObIMemtableCtxFactory*, oceanbase::storage::ObSSTableMergeCtx&, oceanbase::compaction::ObIStoreRowProcessor&, long, bool)

./build_release/src/storage/./src/storage/compaction/ob_partition_merge_util.cpp:1723

oceanbase::storage::ObMacroBlockMergeTask::process()

./build_release/src/storage/./src/storage/ob_partition_merge_task.cpp:144

oceanbase::share::ObITask::do_work()

./build_release/src/share/./src/share/scheduler/ob_dag_scheduler.cpp:89

oceanbase::share::ObDagWorker::run1()

./build_release/src/share/./src/share/scheduler/ob_dag_scheduler.cpp:667

oceanbase::lib::ThreadPool::run0()

./build_release/src/share/./deps/oblib/src/lib/thread/thread_pool.h:45

operator()

./build_release/src/share/./deps/oblib/src/lib/coro/co_user_thread.h:295

oceanbase::lib::CoSetSched::Worker::run()

./build_release/deps/oblib/src/lib/./deps/oblib/src/lib/coro/co_set_sched.cpp:95

oceanbase::lib::CoRoutine::__start(boost::context::detail::transfer_t)

./build_release/deps/oblib/src/lib/./deps/oblib/src/lib/coro/co_routine.cpp:132

finish

/home/zhr/oceanbase/deps/oblib/src/lib/coro/context/asm/make_x86_64_sysv_elf_gas.S:75


#3

配置文件

oceanbase-ce:

 servers:

  • 127.0.0.1

 global:

  home_path: /nvme1n1/ob_data

  devname: lo

  mysql_port: 5881

  rpc_port: 5882

  zone: zone1

  cluster_id: 1

  memory_limit: 150G

  system_memory: 4G

  #datafile_disk_percentage: 10

  datafile_size: 200G

  syslog_level: ERROR

  enable_syslog_recycle: true

  max_syslog_file_count: 4

资源池以及租户配置：

create resource unit tpch_unit max_cpu 112, max_memory '50G', max_iops 128, max_disk_size '150G', max_session_num 64, MIN_CPU=112, MIN_MEMORY '50G', MIN_IOPS=128;

create resource pool tpch_pool unit = 'tpch_unit', unit_num = 1, zone_list=('zone1');

create tenant tpch_mysql resource_pool_list=('tpch_pool'), charset=utf8mb4, replica_num=3, zone_list('zone1'), primary_zone=RANDOM, locality='F@zone1' set variables ob_compatibility_mode='mysql', ob_tcp_invited_nodes='%';


#4

进程CRASH ERROR是指 因为断错误导致的进程崩溃 还是说系统打印了ERROR日志？如果是前者，希望提供一下coredump的堆栈；

#5

CRASH ERROR!!! sig=11, sig_code=1, sig_addr=7f6da82a9e87, timestamp=1648520745574191, tid=45517, tname=DAG_MINOR_MERGE, trace_id=25265128341505-1648520593375852, extra_info=((null)), lbt=0x99eca68 0x99dc8a8 0x7f759eca1c1f 0x84acc7f 0x83fd673 0x8416897 0x84162bb 0x8415714 0x840db21 0x85a3451 0x85b0d2c 0x858b8f9 0x7b6b4a7 0x7b6f7f3 0x7b72a60 0x7c98725 0x3385839 0x3388caf 0x236ef0e 0x23a37a1 0x998c534 0x998b261 0x973526e

这个堆栈可以打一下吗

#6

看起来是打印日志core掉了，调整下日志级别也许可以绕过